diff --git a/indexer/CHANGELOG.md b/indexer/CHANGELOG.md index b1cc352..a48f59b 100644 --- a/indexer/CHANGELOG.md +++ b/indexer/CHANGELOG.md @@ -1,5 +1,19 @@ CHANGELOG --- +0.13.0 +- Added support for `--compare=DB` +- start using `—block=#` instead of `—reparse=#` +- Updated `createTxIndex()` to call on `createAction()` instead of `createTxType()` +- Updated `transactions` table to change `type_id` to `action_id` +- Updated `isValidLock()` so unset lock values are valid +- Updated `getTokenInfo()` so empty values are ignored +- Removed `index_tx_types` table (duplicated via `index_actions`) +- Removed `createTxType()` function +- Updated `getAddressCreditDebit()` to lookup balances using `block_index` or `tx_index` +- Updated `getTokenInfo()` to removed duplicated `OWNER` array item +- Cleanup `MINT_START/STOP_BLOCK` logic +- Updated `--compare=DB` to ignore missing txs in compare ledger + 0.12.0 - Added support for `--reparse` - Optimized ledger hashing diff --git a/indexer/includes/actions/issue.php b/indexer/includes/actions/issue.php index 92d01b1..d2392de 100644 --- a/indexer/includes/actions/issue.php +++ b/indexer/includes/actions/issue.php @@ -268,15 +268,15 @@ function btnsIssue( $params=null, $data=null, $error=null){ $error = 'invalid: BLOCK_LIST (bad list)'; // Verify MINT_START_BLOCK is greater than or equal to current block - if(!$error && isset($data->MINT_START_BLOCK) && $data->MINT_START_BLOCK > 0 && $data->MINT_START_BLOCK < $data->BLOCK_INDEX) + if(!$error && isset($issue->MINT_START_BLOCK) && $issue->MINT_START_BLOCK > 0 && $issue->MINT_START_BLOCK < $issue->BLOCK_INDEX) $error = 'invalid: MINT_START_BLOCK < BLOCK_INDEX'; // Verify MINT_STOP_BLOCK is greater than or equal to current block - if(!$error && isset($data->MINT_STOP_BLOCK) && $data->MINT_STOP_BLOCK > 0 && $data->MINT_STOP_BLOCK < $data->BLOCK_INDEX) + if(!$error && isset($issue->MINT_STOP_BLOCK) && $issue->MINT_STOP_BLOCK > 0 && $issue->MINT_STOP_BLOCK < $issue->BLOCK_INDEX) $error = 'invalid: MINT_STOP_BLOCK < BLOCK_INDEX'; // Verify MINT_STOP_BLOCK is greater than or equal to MINT_START_BLOCK - if(!$error && isset($data->MINT_STOP_BLOCK) && $data->MINT_START_BLOCK > 0 && $data->MINT_STOP_BLOCK > 0 && $data->MINT_STOP_BLOCK < $data->MINT_START_BLOCK) + if(!$error && isset($issue->MINT_STOP_BLOCK) && $issue->MINT_START_BLOCK > 0 && $issue->MINT_STOP_BLOCK > 0 && $issue->MINT_STOP_BLOCK < $issue->MINT_START_BLOCK) $error = 'invalid: MINT_STOP_BLOCK < MINT_START_BLOCK'; // Determine final status diff --git a/indexer/includes/compare.php b/indexer/includes/compare.php new file mode 100644 index 0000000..a3b0454 --- /dev/null +++ b/indexer/includes/compare.php @@ -0,0 +1,187 @@ + DB_DATA, + 'compare' => $database + ); + + // Setup placeholders for ledger transactions + $data = array(); + foreach($ledgers as $name => $db) + $data[$name] = array(); + + // Verify database exists + try { + $database = $mysqli->real_escape_string($database); + $results = $mysqli->query("USE {$database}"); + } catch (Exception $e){ + $error = "database {$database} not found"; + } + + // Build out SQL query based on runtime flags + $whereSql = ''; + $limitSql = ($single) ? 'LIMIT 1' : ''; + if($block){ + $op = ($single) ? '=' : '>='; + $whereSql = " AND block_index {$op} {$block}"; + } + + // Get list of transactions from each ledger + foreach($ledgers as $name => $db){ + if(!$error){ + $data[$name]['transactions'] = array(); + print "Getting data for {$name} ledger from {$db} database...\n"; + $sql = "SELECT + t1.tx_index, + t2.hash as tx_hash, + a.action + FROM + {$db}.transactions t1, + {$db}.index_transactions t2, + {$db}.index_actions a + WHERE + a.id=t1.action_id AND + t2.id=t1.tx_hash_id + {$whereSql} + ORDER BY tx_index ASC + {$limitSql}"; + // Support OLD style database with index_tx_types instead of index_actions + // TODO: Remove + if(in_array($db,array('BTNS_Counterparty_Old','BTNS_Counterparty_Testnet_Old','BTNS_Dogeparty_Old','BTNS_Dogeparty_Testnet_Old'))){ + $sql = "SELECT + t1.tx_index, + t2.hash as tx_hash, + a.type as action + FROM + {$db}.transactions t1, + {$db}.index_transactions t2, + {$db}.index_tx_types a + WHERE + a.id=t1.type_id AND + t2.id=t1.tx_hash_id + {$whereSql} + ORDER BY tx_index ASC + {$limitSql}"; + } + + $results = $mysqli->query($sql); + if($results){ + if($results->num_rows){ + while($row = $results->fetch_assoc()) + $data[$name]['transactions'][$row['tx_index']] = $row; + } + } else { + $error = "Error while looking up transactions in the {$db} database"; + } + } + if(!$error) + print "Found " . number_format(count($data[$name]['transactions'])) . " transactions\n"; + } + + // Loop through ledger transactions + foreach($data['current']['transactions'] as $tx_index => $info){ + + // debug: force mismatches + // $data['compare']['transactions'][1]['tx_hash'] = 'test'; + // $data['compare']['transactions'][1]['action'] = 'test'; + + // Compare basic transaction data (tx_index, tx_hash, action) + foreach(array_keys($info) as $field) + if(!$error && isset($data['compare']['transactions'][$tx_index]) && $data['compare']['transactions'][$tx_index][$field]!=$info[$field]) + $error = "ERROR: Found ledger {$field} difference at tx_index {$tx_index}! ({$info[$field]} != {$data['compare']['transactions'][$tx_index][$field]})"; + + // Lookup transaction statuses for records related to this transaction + if(!$error){ + $action = strtolower($info['action']); + $append = (in_array($action,array('address','batch'))) ? 'es' : 's'; + $table = $action . $append; + if(in_array($table, $tables)){ + + // Handle batches by looping through all tables looking for data + $arr = ($table=='batch') ? $tables : [$table]; + + // Loop through tables with transaction data + foreach($arr as $table){ + foreach($ledgers as $name => $db){ + if(!$error){ + $data[$name]['txinfo'] = array(); + $sql = "SELECT + a.address as source, + s.status + FROM + {$db}.{$table} m, + {$db}.index_statuses s, + {$db}.index_addresses a + WHERE + s.id=m.status_id AND + a.id=m.source_id AND + m.tx_index='{$tx_index}'"; + // print $sql; + $results = $mysqli->query($sql); + if($results){ + if($results->num_rows) + while($row = $results->fetch_assoc()) + array_push($data[$name]['txinfo'], $row); + } else { + $error = "Error while looking up {$table} data in the {$db} database"; + } + } + } + + // debug: force mismatches + // $data['compare']['txinfo'][0]['source'] = 'test'; + // $data['compare']['txinfo'][0]['status'] = 'test'; + + // Compare transaction data (source, status) + if(!$error){ + foreach($data['current']['txinfo'] as $idx => $nfo){ + foreach(array_keys($nfo) as $field) + if(!$error && isset($data['compare']['txinfo'][$idx]) && $data['compare']['txinfo'][$idx][$field]!=$nfo[$field]) + $error = "ERROR: Found ledger {$field} difference at tx_index {$tx_index}! ({$nfo[$field]} != {$data['compare']['txinfo'][$idx][$field]})"; + } + } + } + } else { + // Throw error if table is not found + // $error = "Error table {$table} not found\n"; + } + } + + // Bail out on any error + if($error) + break; + } + + // Display any errors and exit + if($error) + byeLog($error); + + // Print out information on the total runtime + printRuntime($runtime->finish()); + + // Notify user comparison is complete + byeLog("Compare complete."); +} + +?> diff --git a/indexer/includes/config.php b/indexer/includes/config.php index 104393e..2326191 100644 --- a/indexer/includes/config.php +++ b/indexer/includes/config.php @@ -9,7 +9,7 @@ // BTNS Indexer Version define("VERSION_MAJOR", 0); -define("VERSION_MINOR", 12); +define("VERSION_MINOR", 13); define("VERSION_REVISION",0); define("VERSION_STRING", VERSION_MAJOR . '.' . VERSION_MINOR . '.' . VERSION_REVISION); @@ -69,11 +69,10 @@ // Tracks Execution Time require_once('profiler.php'); -// Rollback code +// Ledger Management / Maintenance require_once('rollback.php'); - -// Reparse code require_once('reparse.php'); +require_once('compare.php'); // Protocol Changes / Activation blocks require_once('protocol_changes.php'); diff --git a/indexer/includes/functions.php b/indexer/includes/functions.php index acc4452..52bc348 100644 --- a/indexer/includes/functions.php +++ b/indexer/includes/functions.php @@ -929,7 +929,6 @@ function getTokenInfo($tick=null, $tick_id=null, $block_index=null, $tx_index=nu 'MAX_MINT' => $row->max_mint, 'DECIMALS' => (isset($row->decimals)) ? intval($row->decimals) : 0, 'DESCRIPTION' => $row->description, - 'OWNER' => $row->owner, 'LOCK_MAX_SUPPLY' => $row->lock_max_supply, 'LOCK_MINT_SUPPLY' => $row->lock_mint_supply, 'LOCK_MINT' => $row->lock_mint, @@ -954,8 +953,8 @@ function getTokenInfo($tick=null, $tick_id=null, $block_index=null, $tx_index=nu if(substr($key,0,5)=='LOCK_') if($data[$key]==1) continue; - // Skip setting value if value is null - if(in_array($key,array('MAX_SUPPLY','MAX_MINT')) && !isset($value)) + // Skip setting value if value is null or empty (use last explicit value) + if(!isset($value) || $value=='') continue; $data[$key] = $value; } @@ -1102,10 +1101,12 @@ function getAddressCreditDebit($table=null, $address=null, $action=null, $block= $whereSql = ""; if(isset($action)) $whereSql .= " AND t1.action_id={$action_id}"; - if(isset($block) && is_numeric($block)) - $whereSql .= " AND t1.block_index < {$block}"; - if(isset($tx_index) && is_numeric($tx_index)) + // Query using either block_index OR tx_index + if(isset($tx_index) && is_numeric($tx_index)){ $whereSql .= " AND t3.tx_index < {$tx_index}"; + } else if(isset($block) && is_numeric($block)){ + $whereSql .= " AND t1.block_index < {$block}"; + } if(in_array($table,array('credits','debits'))){ // Get data from the table $sql = "SELECT @@ -1135,8 +1136,7 @@ function getAddressCreditDebit($table=null, $address=null, $action=null, $block= byeLog("Error while trying to lookup address {$table} for : {$address}"); } } - return $data; - + return $data; } // Get address balances using credits/debits table data @@ -1437,28 +1437,6 @@ function btnsAction($action=null, $params=null, $data=null, $error=null){ if($action=='SWEEP') btnsSweep($params, $data, $error); } -// Create records in the 'tx_index_types' table and return record id -function createTxType( $type=null ){ - global $mysqli; - $type = $mysqli->real_escape_string($type); - $results = $mysqli->query("SELECT id FROM index_tx_types WHERE type='{$type}' LIMIT 1"); - if($results){ - if($results->num_rows){ - $row = $results->fetch_assoc(); - return $row['id']; - } else { - $results = $mysqli->query("INSERT INTO index_tx_types (type) values ('{$type}')"); - if($results){ - return $mysqli->insert_id; - } else { - byeLog('Error while trying to create record in index_tx_types table'); - } - } - } else { - byeLog('Error while trying to lookup record in index_tx_types table'); - } -} - // Handles returning the highest tx_index from transactions table function getNextTxIndex(){ global $mysqli; @@ -1493,12 +1471,12 @@ function createTxIndex( $data=null ){ // Get highest tx_index $block_index = $data->BLOCK_INDEX; $tx_hash_id = createTransaction($data->TX_HASH); - $type_id = createTxType($data->ACTION); + $action_id = createAction($data->ACTION); $tx_index = getNextTxIndex(); - $results = $mysqli->query("SELECT type_id FROM transactions WHERE tx_hash_id='{$tx_hash_id}' LIMIT 1"); + $results = $mysqli->query("SELECT action_id FROM transactions WHERE tx_hash_id='{$tx_hash_id}' LIMIT 1"); if($results){ if($results->num_rows==0){ - $results = $mysqli->query("INSERT INTO transactions (tx_index, block_index, tx_hash_id, type_id) values ('{$tx_index}','{$block_index}','{$tx_hash_id}', '{$type_id}')"); + $results = $mysqli->query("INSERT INTO transactions (tx_index, block_index, tx_hash_id, action_id) values ('{$tx_index}','{$block_index}','{$tx_hash_id}', '{$action_id}')"); if(!$results) byeLog('Error while trying to create record in transactions table'); } @@ -1606,6 +1584,9 @@ function isValidLock($btInfo=null, $data=null, $lock=null){ // If we dont have any info on the token, it hasn't been created yet, so all flags are valid if(!isset($btInfo)) return true; + // If token exists and lock value does not exist yet, its valid + if($btInfo->{$lock}=="") + return true; // If lock value is not changing, its valid if(isset($value) && $btInfo->{$lock}==$value) return true; diff --git a/indexer/includes/rollback.php b/indexer/includes/rollback.php index 5245e3f..a83654e 100644 --- a/indexer/includes/rollback.php +++ b/indexer/includes/rollback.php @@ -3,7 +3,7 @@ * rollback.php - Handles rolling back database updates safely ********************************************************************/ function btnsRollback($block_index=null){ - global $mysqli, $addresses, $tickers; + global $mysqli, $runtime, $addresses, $tickers; $block_index = (int) $block_index; @@ -152,9 +152,8 @@ function btnsRollback($block_index=null){ // Delete items from list_{items,edits} tables deleteLists($transactions, true); - // Report time to process block - $time = $timer->finish(); - print " Done [{$time}sec]\n"; + // Print out information on the total runtime + printRuntime($runtime->finish()); // Notify user rollback is complete byeLog("Rollback to block {$block_index} complete."); diff --git a/indexer/indexer.php b/indexer/indexer.php index 7217cc8..93186a6 100755 --- a/indexer/indexer.php +++ b/indexer/indexer.php @@ -24,8 +24,7 @@ * --testnet Load data from testnet * --block=# Load data for given block * --rollback=# Rollback data to a given block - * --reparse Reparse ALL data - * --reparse=# Reparse data from a given block + * --reparse Reparse transaction data * --single Load single block ********************************************************************/ @@ -33,10 +32,11 @@ error_reporting(E_ERROR|E_PARSE); // Parse in any command line args and set basic runtime flags -$args = getopt("", array("testnet::", "block::", "single::", "rollback::", "reparse::")); +$args = getopt("", array("testnet::", "block::", "single::", "rollback::", "reparse::", "compare::")); $testnet = (isset($args['testnet'])) ? true : false; -$single = (isset($args['single'])) ? true : false; -$reparse = (isset($args['reparse'])) ? ((is_numeric($args['reparse'])) ? $args['reparse'] : true) : false; +$single = (isset($args['single'])) ? true : false; +$reparse = (isset($args['reparse'])) ? true : false; +$compare = (isset($args['compare'])) ? $args['compare'] : false; $block = (is_numeric($args['block'])) ? intval($args['block']) : false; $network = ($testnet) ? 'testnet' : 'mainnet'; $rollback = (is_numeric($args['rollback'])) ? intval($args['rollback']) : false; @@ -70,11 +70,18 @@ if($rollback) btnsRollback($rollback); +// Handle compares +if($compare) + btnsCompare($compare); + // If no block given, load last block from state file, or use first block with BTNX tx if(!$block){ $last = file_get_contents(LASTFILE); $first = FIRST_BLOCK; // First block a BTNS transaction is seen $block = (isset($last) && $last>=$first) ? (intval($last) + 1) : $first; + // If not reparse block is given, start reparse at first block + if($reparse) + $block = $first; } // Get the current block index from status info @@ -94,7 +101,7 @@ // Handle reparses if($reparse) - btnsReparse($current, $reparse); + btnsReparse($current, $block); // Loop through the blocks until we are current while($block <= $current){ diff --git a/indexer/sql/index_tx_types.sql b/indexer/sql/index_tx_types.sql deleted file mode 100644 index 0c0d4a7..0000000 --- a/indexer/sql/index_tx_types.sql +++ /dev/null @@ -1,7 +0,0 @@ -DROP TABLE IF EXISTS index_tx_types; -CREATE TABLE index_tx_types ( - id INTEGER UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, - type VARCHAR(100) NOT NULL -) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; - -CREATE INDEX type on index_tx_types (type); diff --git a/indexer/sql/transactions.sql b/indexer/sql/transactions.sql index fe215ec..604021d 100644 --- a/indexer/sql/transactions.sql +++ b/indexer/sql/transactions.sql @@ -2,11 +2,11 @@ DROP TABLE IF EXISTS transactions; CREATE TABLE transactions ( tx_index INTEGER UNSIGNED NOT NULL, block_index INTEGER, - tx_hash_id INTEGER UNSIGNED NOT NULL, - type_id INTEGER UNSIGNED NOT NULL -- id of record in index_tx_types table + tx_hash_id INTEGER UNSIGNED NOT NULL, -- id of record in index_transactions table + action_id INTEGER UNSIGNED NOT NULL -- id of record in index_actions table ) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; CREATE UNIQUE INDEX tx_index on transactions (tx_index); CREATE INDEX block_index on transactions (block_index); CREATE INDEX tx_hash_id on transactions (tx_hash_id); -CREATE INDEX type_id on transactions (type_id); +CREATE INDEX action_id on transactions (action_id);