Skip to content

Commit

Permalink
Merge pull request #38 from jdogresorg/btns420-indexer
Browse files Browse the repository at this point in the history
v0.13.0 Release
  • Loading branch information
jdogresorg authored Apr 19, 2024
2 parents b2903f5 + 3ce4e78 commit 0f2cd66
Show file tree
Hide file tree
Showing 9 changed files with 240 additions and 60 deletions.
14 changes: 14 additions & 0 deletions indexer/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,19 @@
CHANGELOG
---
0.13.0
- Added support for `--compare=DB`
- start using `—block=#` instead of `—reparse=#`
- Updated `createTxIndex()` to call on `createAction()` instead of `createTxType()`
- Updated `transactions` table to change `type_id` to `action_id`
- Updated `isValidLock()` so unset lock values are valid
- Updated `getTokenInfo()` so empty values are ignored
- Removed `index_tx_types` table (duplicated via `index_actions`)
- Removed `createTxType()` function
- Updated `getAddressCreditDebit()` to lookup balances using `block_index` or `tx_index`
- Updated `getTokenInfo()` to removed duplicated `OWNER` array item
- Cleanup `MINT_START/STOP_BLOCK` logic
- Updated `--compare=DB` to ignore missing txs in compare ledger

0.12.0
- Added support for `--reparse`
- Optimized ledger hashing
Expand Down
6 changes: 3 additions & 3 deletions indexer/includes/actions/issue.php
Original file line number Diff line number Diff line change
Expand Up @@ -268,15 +268,15 @@ function btnsIssue( $params=null, $data=null, $error=null){
$error = 'invalid: BLOCK_LIST (bad list)';

// Verify MINT_START_BLOCK is greater than or equal to current block
if(!$error && isset($data->MINT_START_BLOCK) && $data->MINT_START_BLOCK > 0 && $data->MINT_START_BLOCK < $data->BLOCK_INDEX)
if(!$error && isset($issue->MINT_START_BLOCK) && $issue->MINT_START_BLOCK > 0 && $issue->MINT_START_BLOCK < $issue->BLOCK_INDEX)
$error = 'invalid: MINT_START_BLOCK < BLOCK_INDEX';

// Verify MINT_STOP_BLOCK is greater than or equal to current block
if(!$error && isset($data->MINT_STOP_BLOCK) && $data->MINT_STOP_BLOCK > 0 && $data->MINT_STOP_BLOCK < $data->BLOCK_INDEX)
if(!$error && isset($issue->MINT_STOP_BLOCK) && $issue->MINT_STOP_BLOCK > 0 && $issue->MINT_STOP_BLOCK < $issue->BLOCK_INDEX)
$error = 'invalid: MINT_STOP_BLOCK < BLOCK_INDEX';

// Verify MINT_STOP_BLOCK is greater than or equal to MINT_START_BLOCK
if(!$error && isset($data->MINT_STOP_BLOCK) && $data->MINT_START_BLOCK > 0 && $data->MINT_STOP_BLOCK > 0 && $data->MINT_STOP_BLOCK < $data->MINT_START_BLOCK)
if(!$error && isset($issue->MINT_STOP_BLOCK) && $issue->MINT_START_BLOCK > 0 && $issue->MINT_STOP_BLOCK > 0 && $issue->MINT_STOP_BLOCK < $issue->MINT_START_BLOCK)
$error = 'invalid: MINT_STOP_BLOCK < MINT_START_BLOCK';

// Determine final status
Expand Down
187 changes: 187 additions & 0 deletions indexer/includes/compare.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
<?php
/*********************************************************************
* compare.php - Handles comparing ledger against a given database
********************************************************************/
function btnsCompare($database=null){
global $mysqli, $block, $single, $runtime;

// Placeholder for any error message
$error = false;

// Array of tables containing ledger data
$tables = [
'addresses',
'airdrops',
'batches',
'destroys',
'issues',
'lists',
'mints',
'sends'
];

// Array of ledgers to compare
$ledgers = array (
'current' => DB_DATA,
'compare' => $database
);

// Setup placeholders for ledger transactions
$data = array();
foreach($ledgers as $name => $db)
$data[$name] = array();

// Verify database exists
try {
$database = $mysqli->real_escape_string($database);
$results = $mysqli->query("USE {$database}");
} catch (Exception $e){
$error = "database {$database} not found";
}

// Build out SQL query based on runtime flags
$whereSql = '';
$limitSql = ($single) ? 'LIMIT 1' : '';
if($block){
$op = ($single) ? '=' : '>=';
$whereSql = " AND block_index {$op} {$block}";
}

// Get list of transactions from each ledger
foreach($ledgers as $name => $db){
if(!$error){
$data[$name]['transactions'] = array();
print "Getting data for {$name} ledger from {$db} database...\n";
$sql = "SELECT
t1.tx_index,
t2.hash as tx_hash,
a.action
FROM
{$db}.transactions t1,
{$db}.index_transactions t2,
{$db}.index_actions a
WHERE
a.id=t1.action_id AND
t2.id=t1.tx_hash_id
{$whereSql}
ORDER BY tx_index ASC
{$limitSql}";
// Support OLD style database with index_tx_types instead of index_actions
// TODO: Remove
if(in_array($db,array('BTNS_Counterparty_Old','BTNS_Counterparty_Testnet_Old','BTNS_Dogeparty_Old','BTNS_Dogeparty_Testnet_Old'))){
$sql = "SELECT
t1.tx_index,
t2.hash as tx_hash,
a.type as action
FROM
{$db}.transactions t1,
{$db}.index_transactions t2,
{$db}.index_tx_types a
WHERE
a.id=t1.type_id AND
t2.id=t1.tx_hash_id
{$whereSql}
ORDER BY tx_index ASC
{$limitSql}";
}

$results = $mysqli->query($sql);
if($results){
if($results->num_rows){
while($row = $results->fetch_assoc())
$data[$name]['transactions'][$row['tx_index']] = $row;
}
} else {
$error = "Error while looking up transactions in the {$db} database";
}
}
if(!$error)
print "Found " . number_format(count($data[$name]['transactions'])) . " transactions\n";
}

// Loop through ledger transactions
foreach($data['current']['transactions'] as $tx_index => $info){

// debug: force mismatches
// $data['compare']['transactions'][1]['tx_hash'] = 'test';
// $data['compare']['transactions'][1]['action'] = 'test';

// Compare basic transaction data (tx_index, tx_hash, action)
foreach(array_keys($info) as $field)
if(!$error && isset($data['compare']['transactions'][$tx_index]) && $data['compare']['transactions'][$tx_index][$field]!=$info[$field])
$error = "ERROR: Found ledger {$field} difference at tx_index {$tx_index}! ({$info[$field]} != {$data['compare']['transactions'][$tx_index][$field]})";

// Lookup transaction statuses for records related to this transaction
if(!$error){
$action = strtolower($info['action']);
$append = (in_array($action,array('address','batch'))) ? 'es' : 's';
$table = $action . $append;
if(in_array($table, $tables)){

// Handle batches by looping through all tables looking for data
$arr = ($table=='batch') ? $tables : [$table];

// Loop through tables with transaction data
foreach($arr as $table){
foreach($ledgers as $name => $db){
if(!$error){
$data[$name]['txinfo'] = array();
$sql = "SELECT
a.address as source,
s.status
FROM
{$db}.{$table} m,
{$db}.index_statuses s,
{$db}.index_addresses a
WHERE
s.id=m.status_id AND
a.id=m.source_id AND
m.tx_index='{$tx_index}'";
// print $sql;
$results = $mysqli->query($sql);
if($results){
if($results->num_rows)
while($row = $results->fetch_assoc())
array_push($data[$name]['txinfo'], $row);
} else {
$error = "Error while looking up {$table} data in the {$db} database";
}
}
}

// debug: force mismatches
// $data['compare']['txinfo'][0]['source'] = 'test';
// $data['compare']['txinfo'][0]['status'] = 'test';

// Compare transaction data (source, status)
if(!$error){
foreach($data['current']['txinfo'] as $idx => $nfo){
foreach(array_keys($nfo) as $field)
if(!$error && isset($data['compare']['txinfo'][$idx]) && $data['compare']['txinfo'][$idx][$field]!=$nfo[$field])
$error = "ERROR: Found ledger {$field} difference at tx_index {$tx_index}! ({$nfo[$field]} != {$data['compare']['txinfo'][$idx][$field]})";
}
}
}
} else {
// Throw error if table is not found
// $error = "Error table {$table} not found\n";
}
}

// Bail out on any error
if($error)
break;
}

// Display any errors and exit
if($error)
byeLog($error);

// Print out information on the total runtime
printRuntime($runtime->finish());

// Notify user comparison is complete
byeLog("Compare complete.");
}

?>
7 changes: 3 additions & 4 deletions indexer/includes/config.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

// BTNS Indexer Version
define("VERSION_MAJOR", 0);
define("VERSION_MINOR", 12);
define("VERSION_MINOR", 13);
define("VERSION_REVISION",0);
define("VERSION_STRING", VERSION_MAJOR . '.' . VERSION_MINOR . '.' . VERSION_REVISION);

Expand Down Expand Up @@ -69,11 +69,10 @@
// Tracks Execution Time
require_once('profiler.php');

// Rollback code
// Ledger Management / Maintenance
require_once('rollback.php');

// Reparse code
require_once('reparse.php');
require_once('compare.php');

// Protocol Changes / Activation blocks
require_once('protocol_changes.php');
Expand Down
47 changes: 14 additions & 33 deletions indexer/includes/functions.php
Original file line number Diff line number Diff line change
Expand Up @@ -929,7 +929,6 @@ function getTokenInfo($tick=null, $tick_id=null, $block_index=null, $tx_index=nu
'MAX_MINT' => $row->max_mint,
'DECIMALS' => (isset($row->decimals)) ? intval($row->decimals) : 0,
'DESCRIPTION' => $row->description,
'OWNER' => $row->owner,
'LOCK_MAX_SUPPLY' => $row->lock_max_supply,
'LOCK_MINT_SUPPLY' => $row->lock_mint_supply,
'LOCK_MINT' => $row->lock_mint,
Expand All @@ -954,8 +953,8 @@ function getTokenInfo($tick=null, $tick_id=null, $block_index=null, $tx_index=nu
if(substr($key,0,5)=='LOCK_')
if($data[$key]==1)
continue;
// Skip setting value if value is null
if(in_array($key,array('MAX_SUPPLY','MAX_MINT')) && !isset($value))
// Skip setting value if value is null or empty (use last explicit value)
if(!isset($value) || $value=='')
continue;
$data[$key] = $value;
}
Expand Down Expand Up @@ -1102,10 +1101,12 @@ function getAddressCreditDebit($table=null, $address=null, $action=null, $block=
$whereSql = "";
if(isset($action))
$whereSql .= " AND t1.action_id={$action_id}";
if(isset($block) && is_numeric($block))
$whereSql .= " AND t1.block_index < {$block}";
if(isset($tx_index) && is_numeric($tx_index))
// Query using either block_index OR tx_index
if(isset($tx_index) && is_numeric($tx_index)){
$whereSql .= " AND t3.tx_index < {$tx_index}";
} else if(isset($block) && is_numeric($block)){
$whereSql .= " AND t1.block_index < {$block}";
}
if(in_array($table,array('credits','debits'))){
// Get data from the table
$sql = "SELECT
Expand Down Expand Up @@ -1135,8 +1136,7 @@ function getAddressCreditDebit($table=null, $address=null, $action=null, $block=
byeLog("Error while trying to lookup address {$table} for : {$address}");
}
}
return $data;

return $data;
}

// Get address balances using credits/debits table data
Expand Down Expand Up @@ -1437,28 +1437,6 @@ function btnsAction($action=null, $params=null, $data=null, $error=null){
if($action=='SWEEP') btnsSweep($params, $data, $error);
}

// Create records in the 'tx_index_types' table and return record id
function createTxType( $type=null ){
global $mysqli;
$type = $mysqli->real_escape_string($type);
$results = $mysqli->query("SELECT id FROM index_tx_types WHERE type='{$type}' LIMIT 1");
if($results){
if($results->num_rows){
$row = $results->fetch_assoc();
return $row['id'];
} else {
$results = $mysqli->query("INSERT INTO index_tx_types (type) values ('{$type}')");
if($results){
return $mysqli->insert_id;
} else {
byeLog('Error while trying to create record in index_tx_types table');
}
}
} else {
byeLog('Error while trying to lookup record in index_tx_types table');
}
}

// Handles returning the highest tx_index from transactions table
function getNextTxIndex(){
global $mysqli;
Expand Down Expand Up @@ -1493,12 +1471,12 @@ function createTxIndex( $data=null ){
// Get highest tx_index
$block_index = $data->BLOCK_INDEX;
$tx_hash_id = createTransaction($data->TX_HASH);
$type_id = createTxType($data->ACTION);
$action_id = createAction($data->ACTION);
$tx_index = getNextTxIndex();
$results = $mysqli->query("SELECT type_id FROM transactions WHERE tx_hash_id='{$tx_hash_id}' LIMIT 1");
$results = $mysqli->query("SELECT action_id FROM transactions WHERE tx_hash_id='{$tx_hash_id}' LIMIT 1");
if($results){
if($results->num_rows==0){
$results = $mysqli->query("INSERT INTO transactions (tx_index, block_index, tx_hash_id, type_id) values ('{$tx_index}','{$block_index}','{$tx_hash_id}', '{$type_id}')");
$results = $mysqli->query("INSERT INTO transactions (tx_index, block_index, tx_hash_id, action_id) values ('{$tx_index}','{$block_index}','{$tx_hash_id}', '{$action_id}')");
if(!$results)
byeLog('Error while trying to create record in transactions table');
}
Expand Down Expand Up @@ -1606,6 +1584,9 @@ function isValidLock($btInfo=null, $data=null, $lock=null){
// If we dont have any info on the token, it hasn't been created yet, so all flags are valid
if(!isset($btInfo))
return true;
// If token exists and lock value does not exist yet, its valid
if($btInfo->{$lock}=="")
return true;
// If lock value is not changing, its valid
if(isset($value) && $btInfo->{$lock}==$value)
return true;
Expand Down
7 changes: 3 additions & 4 deletions indexer/includes/rollback.php
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* rollback.php - Handles rolling back database updates safely
********************************************************************/
function btnsRollback($block_index=null){
global $mysqli, $addresses, $tickers;
global $mysqli, $runtime, $addresses, $tickers;

$block_index = (int) $block_index;

Expand Down Expand Up @@ -152,9 +152,8 @@ function btnsRollback($block_index=null){
// Delete items from list_{items,edits} tables
deleteLists($transactions, true);

// Report time to process block
$time = $timer->finish();
print " Done [{$time}sec]\n";
// Print out information on the total runtime
printRuntime($runtime->finish());

// Notify user rollback is complete
byeLog("Rollback to block {$block_index} complete.");
Expand Down
Loading

0 comments on commit 0f2cd66

Please sign in to comment.