From 1236400c4b392b73e9a79e05219b7bed6bfcbae6 Mon Sep 17 00:00:00 2001 From: Boris Oncev Date: Thu, 16 May 2024 13:23:19 +0200 Subject: [PATCH 1/2] port more changes from master into release 0.4.3 --- .../src/storage/impls/in_memory/mod.rs | 34 +++---- .../src/storage/impls/postgres/queries.rs | 62 +++++++++---- .../src/storage/storage_api/mod.rs | 9 ++ .../scanner-lib/src/blockchain_state/mod.rs | 93 ++++++++++++------- .../tests/v2/address_all_utxos.rs | 10 -- api-server/storage-test-suite/src/basic.rs | 13 +-- 6 files changed, 135 insertions(+), 86 deletions(-) diff --git a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs index 4c9ef9d369..89d5a0eb99 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs @@ -450,23 +450,23 @@ impl ApiServerInMemoryStorage { .get(address) .unwrap_or(&BTreeSet::new()) .union(self.address_locked_utxos.get(address).unwrap_or(&BTreeSet::new())) - .map(|outpoint| { - ( - outpoint.clone(), - self.get_utxo(outpoint.clone()).expect("no error").map_or_else( - || { - self.locked_utxo_table - .get(outpoint) - .expect("must exit") - .values() - .last() - .expect("not empty") - .utxo_with_extra_info() - .clone() - }, - |utxo| utxo.utxo_with_extra_info().clone(), - ), - ) + .filter_map(|outpoint| { + if let Some(utxo) = self.get_utxo(outpoint.clone()).expect("no error") { + (!utxo.spent()) + .then_some((outpoint.clone(), utxo.utxo_with_extra_info().clone())) + } else { + Some(( + outpoint.clone(), + self.locked_utxo_table + .get(outpoint) + .expect("must exit") + .values() + .last() + .expect("not empty") + .utxo_with_extra_info() + .clone(), + )) + } }) .collect(); Ok(result) diff --git a/api-server/api-server-common/src/storage/impls/postgres/queries.rs b/api-server/api-server-common/src/storage/impls/postgres/queries.rs index 72f4a4f4cb..b792abee15 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/queries.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/queries.rs @@ -589,6 +589,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { "CREATE TABLE ml.delegations ( delegation_id TEXT NOT NULL, block_height bigint NOT NULL, + creation_block_height bigint NOT NULL, pool_id TEXT NOT NULL, balance TEXT NOT NULL, next_nonce bytea NOT NULL, @@ -846,7 +847,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { let row = self .tx .query_opt( - r#"SELECT pool_id, balance, spend_destination, next_nonce + r#"SELECT pool_id, balance, spend_destination, next_nonce, creation_block_height FROM ml.delegations WHERE delegation_id = $1 AND block_height = (SELECT MAX(block_height) FROM ml.delegations WHERE delegation_id = $1); @@ -868,6 +869,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { let balance: String = data.get(1); let spend_destination: Vec = data.get(2); let next_nonce: Vec = data.get(3); + let creation_block_height: i64 = data.get(4); let balance = Amount::from_fixedpoint_str(&balance, 0).ok_or_else(|| { ApiServerStorageError::DeserializationError(format!( @@ -890,7 +892,13 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { )) })?; - let delegation = Delegation::new(spend_destination, pool_id, balance, next_nonce); + let delegation = Delegation::new( + BlockHeight::new(creation_block_height as u64), + spend_destination, + pool_id, + balance, + next_nonce, + ); Ok(Some(delegation)) } @@ -902,9 +910,9 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { let rows = self .tx .query( - r#"SELECT delegation_id, pool_id, balance, spend_destination, next_nonce + r#"SELECT delegation_id, pool_id, balance, spend_destination, next_nonce, creation_block_height FROM ( - SELECT delegation_id, pool_id, balance, spend_destination, next_nonce, ROW_NUMBER() OVER(PARTITION BY delegation_id ORDER BY block_height DESC) as newest + SELECT delegation_id, pool_id, balance, spend_destination, next_nonce, creation_block_height, ROW_NUMBER() OVER(PARTITION BY delegation_id ORDER BY block_height DESC) as newest FROM ml.delegations WHERE spend_destination = $1 ) AS sub @@ -929,6 +937,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { let balance: String = row.get(2); let spend_destination: Vec = row.get(3); let next_nonce: Vec = row.get(4); + let creation_block_height: i64 = row.get(5); let balance = Amount::from_fixedpoint_str(&balance, 0).ok_or_else(|| { ApiServerStorageError::DeserializationError(format!( @@ -950,7 +959,13 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { )) })?; - let delegation = Delegation::new(spend_destination, pool_id, balance, next_nonce); + let delegation = Delegation::new( + BlockHeight::new(creation_block_height as u64), + spend_destination, + pool_id, + balance, + next_nonce, + ); Ok((delegation_id, delegation)) }) .collect() @@ -964,6 +979,8 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { chain_config: &ChainConfig, ) -> Result<(), ApiServerStorageError> { let height = Self::block_height_to_postgres_friendly(block_height); + let creation_block_height = + Self::block_height_to_postgres_friendly(delegation.creation_block_height()); let pool_id = Address::new(chain_config, *delegation.pool_id()) .map_err(|_| ApiServerStorageError::AddressableError)?; let delegation_id = Address::new(chain_config, delegation_id) @@ -972,10 +989,10 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { self.tx .execute( r#" - INSERT INTO ml.delegations (delegation_id, block_height, pool_id, balance, spend_destination, next_nonce) - VALUES($1, $2, $3, $4, $5, $6) + INSERT INTO ml.delegations (delegation_id, block_height, pool_id, balance, spend_destination, next_nonce, creation_block_height) + VALUES($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (delegation_id, block_height) DO UPDATE - SET pool_id = $3, balance = $4, spend_destination = $5, next_nonce = $6; + SET pool_id = $3, balance = $4, spend_destination = $5, next_nonce = $6, creation_block_height = $7; "#, &[ &delegation_id.as_str(), @@ -984,6 +1001,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { &amount_to_str(*delegation.balance()), &delegation.spend_destination().encode(), &delegation.next_nonce().encode(), + &creation_block_height, ], ) .await @@ -1065,7 +1083,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .map_err(|_| ApiServerStorageError::AddressableError)?; self.tx .query( - r#"SELECT delegation_id, balance, spend_destination, next_nonce + r#"SELECT delegation_id, balance, spend_destination, next_nonce, creation_block_height FROM ml.delegations WHERE pool_id = $1 AND (delegation_id, block_height) in (SELECT delegation_id, MAX(block_height) @@ -1087,6 +1105,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { let balance: String = row.get(1); let spend_destination: Vec = row.get(2); let next_nonce: Vec = row.get(3); + let creation_block_height: i64 = row.get(4); let balance = Amount::from_fixedpoint_str(&balance, 0).ok_or_else(|| { ApiServerStorageError::DeserializationError(format!( @@ -1110,7 +1129,13 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { Ok(( delegation_id, - Delegation::new(spend_destination, pool_id, balance, next_nonce), + Delegation::new( + BlockHeight::new(creation_block_height as u64), + spend_destination, + pool_id, + balance, + next_nonce, + ), )) }) .collect() @@ -1526,14 +1551,17 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { let rows = self .tx .query( - r#" - SELECT outpoint, utxo - FROM ml.utxo - WHERE address = $1 - UNION + r#"SELECT outpoint, utxo + FROM ( + SELECT outpoint, utxo, spent, ROW_NUMBER() OVER(PARTITION BY outpoint ORDER BY block_height DESC) as newest + FROM ml.utxo + WHERE address = $1 + ) AS sub + WHERE newest = 1 AND spent = false + UNION ALL SELECT outpoint, utxo - FROM ml.locked_utxo - WHERE address = $1 + FROM ml.locked_utxo AS locked + WHERE locked.address = $1 AND NOT EXISTS (SELECT 1 FROM ml.utxo WHERE outpoint = locked.outpoint) ;"#, &[&address], ) diff --git a/api-server/api-server-common/src/storage/storage_api/mod.rs b/api-server/api-server-common/src/storage/storage_api/mod.rs index e6f7fcec7b..40e151992f 100644 --- a/api-server/api-server-common/src/storage/storage_api/mod.rs +++ b/api-server/api-server-common/src/storage/storage_api/mod.rs @@ -66,6 +66,7 @@ pub enum ApiServerStorageError { #[derive(Debug, Eq, PartialEq, Clone, Encode, Decode)] pub struct Delegation { + creation_block_height: BlockHeight, spend_destination: Destination, pool_id: PoolId, balance: Amount, @@ -74,12 +75,14 @@ pub struct Delegation { impl Delegation { pub fn new( + creation_block_height: BlockHeight, spend_destination: Destination, pool_id: PoolId, balance: Amount, next_nonce: AccountNonce, ) -> Self { Self { + creation_block_height, spend_destination, pool_id, balance, @@ -87,6 +90,10 @@ impl Delegation { } } + pub fn creation_block_height(&self) -> BlockHeight { + self.creation_block_height + } + pub fn spend_destination(&self) -> &Destination { &self.spend_destination } @@ -109,6 +116,7 @@ impl Delegation { pool_id: self.pool_id, balance: (self.balance + rewards).expect("no overflow"), next_nonce: self.next_nonce, + creation_block_height: self.creation_block_height, } } @@ -118,6 +126,7 @@ impl Delegation { pool_id: self.pool_id, balance: (self.balance - amount).expect("not underflow"), next_nonce: nonce.increment().expect("no overflow"), + creation_block_height: self.creation_block_height, } } } diff --git a/api-server/scanner-lib/src/blockchain_state/mod.rs b/api-server/scanner-lib/src/blockchain_state/mod.rs index f29ff7bae7..2471b9f3f8 100644 --- a/api-server/scanner-lib/src/blockchain_state/mod.rs +++ b/api-server/scanner-lib/src/blockchain_state/mod.rs @@ -167,7 +167,7 @@ impl LocalBlockchainState for BlockchainState update_tables_from_transaction( Arc::clone(&self.chain_config), &mut db_tx, - block_height, + (block_height, block_timestamp), new_median_time, tx, ) @@ -383,6 +383,7 @@ async fn update_tables_from_block_reward( block_id: Id, ) -> Result<(), ApiServerStorageError> { for (idx, output) in block_rewards.iter().enumerate() { + let outpoint = UtxoOutPoint::new(OutPointSourceId::BlockReward(block_id), idx as u32); match output { TxOutput::Burn(_) | TxOutput::CreateDelegationId(_, _) @@ -392,9 +393,9 @@ async fn update_tables_from_block_reward( | TxOutput::IssueNft(_, _, _) => {} TxOutput::ProduceBlockFromStake(_, _) => { set_utxo( - OutPointSourceId::BlockReward(block_id), - idx, + outpoint, output, + None, db_tx, block_height, false, @@ -410,9 +411,9 @@ async fn update_tables_from_block_reward( .await .expect("unable to update pool data"); set_utxo( - OutPointSourceId::BlockReward(block_id), - idx, + outpoint, output, + None, db_tx, block_height, false, @@ -424,8 +425,8 @@ async fn update_tables_from_block_reward( | TxOutput::LockThenTransfer(output_value, destination, _) => { let address = Address::::new(&chain_config, destination.clone()) .expect("Unable to encode destination"); - match output_value { - OutputValue::TokenV0(_) => {} + let token_decimals = match output_value { + OutputValue::TokenV0(_) => None, OutputValue::TokenV1(token_id, amount) => { increase_address_amount( db_tx, @@ -435,6 +436,7 @@ async fn update_tables_from_block_reward( block_height, ) .await; + Some(token_decimals(*token_id, &BTreeMap::new(), db_tx).await?.1) } OutputValue::Coin(amount) => { increase_address_amount( @@ -445,12 +447,13 @@ async fn update_tables_from_block_reward( block_height, ) .await; + None } - } + }; set_utxo( - OutPointSourceId::BlockReward(block_id), - idx, + outpoint, output, + token_decimals, db_tx, block_height, false, @@ -710,9 +713,9 @@ async fn update_tables_from_consensus_data( let utxo = db_tx.get_utxo(outpoint.clone()).await?.expect("must be present"); set_utxo( - outpoint.source_id(), - outpoint.output_index() as usize, + outpoint.clone(), utxo.output(), + None, db_tx, block_height, true, @@ -774,7 +777,7 @@ async fn update_tables_from_consensus_data( async fn update_tables_from_transaction( chain_config: Arc, db_tx: &mut T, - block_height: BlockHeight, + (block_height, block_timestamp): (BlockHeight, BlockTimestamp), median_time: BlockTimestamp, transaction: &SignedTransaction, ) -> Result<(), ApiServerStorageError> { @@ -791,7 +794,7 @@ async fn update_tables_from_transaction( update_tables_from_transaction_outputs( Arc::clone(&chain_config), db_tx, - block_height, + (block_height, block_timestamp), median_time, transaction.transaction().get_id(), transaction.transaction().inputs(), @@ -887,9 +890,9 @@ async fn update_tables_from_transaction_inputs( OutPointSourceId::BlockReward(_) => { let utxo = db_tx.get_utxo(outpoint.clone()).await?.expect("must be present"); set_utxo( - outpoint.source_id(), - outpoint.output_index() as usize, + outpoint.clone(), utxo.output(), + utxo.utxo_with_extra_info().token_decimals, db_tx, block_height, true, @@ -935,9 +938,9 @@ async fn update_tables_from_transaction_inputs( OutPointSourceId::Transaction(_) => { let utxo = db_tx.get_utxo(outpoint.clone()).await?.expect("must be present"); set_utxo( - outpoint.source_id(), - outpoint.output_index() as usize, + outpoint.clone(), utxo.output(), + utxo.utxo_with_extra_info().token_decimals, db_tx, block_height, true, @@ -950,9 +953,20 @@ async fn update_tables_from_transaction_inputs( | TxOutput::DelegateStaking(_, _) | TxOutput::Burn(_) | TxOutput::DataDeposit(_) - | TxOutput::IssueFungibleToken(_) - | TxOutput::CreateStakePool(_, _) - | TxOutput::ProduceBlockFromStake(_, _) => {} + | TxOutput::IssueFungibleToken(_) => {} + TxOutput::CreateStakePool(pool_id, _) + | TxOutput::ProduceBlockFromStake(_, pool_id) => { + let pool_data = db_tx + .get_pool_data(pool_id) + .await? + .expect("pool data should exist") + .decommission_pool(); + + db_tx + .set_pool_data_at_height(pool_id, &pool_data, block_height) + .await + .expect("unable to update pool data"); + } TxOutput::IssueNft(token_id, _, destination) => { let address = Address::::new(&chain_config, destination) .expect("Unable to encode destination"); @@ -1032,7 +1046,7 @@ async fn update_tables_from_transaction_inputs( async fn update_tables_from_transaction_outputs( chain_config: Arc, db_tx: &mut T, - block_height: BlockHeight, + (block_height, block_timestamp): (BlockHeight, BlockTimestamp), median_time: BlockTimestamp, transaction_id: Id, inputs: &[TxInput], @@ -1042,6 +1056,7 @@ async fn update_tables_from_transaction_outputs( BTreeMap::new(); for (idx, output) in outputs.iter().enumerate() { + let outpoint = UtxoOutPoint::new(OutPointSourceId::Transaction(transaction_id), idx as u32); match output { TxOutput::Burn(_) | TxOutput::DataDeposit(_) => {} TxOutput::IssueFungibleToken(issuance) => { @@ -1076,9 +1091,9 @@ async fn update_tables_from_transaction_outputs( db_tx.set_nft_token_issuance(*token_id, block_height, *issuance.clone()).await?; set_utxo( - OutPointSourceId::Transaction(transaction_id), - idx, + outpoint, output, + None, db_tx, block_height, false, @@ -1094,6 +1109,7 @@ async fn update_tables_from_transaction_outputs( .set_delegation_at_height( make_delegation_id(input0_outpoint), &Delegation::new( + block_height, destination.clone(), *pool_id, Amount::ZERO, @@ -1114,9 +1130,9 @@ async fn update_tables_from_transaction_outputs( .await .expect("Unable to update pool balance"); set_utxo( - OutPointSourceId::Transaction(transaction_id), - idx, + outpoint, output, + None, db_tx, block_height, false, @@ -1208,7 +1224,7 @@ async fn update_tables_from_transaction_outputs( let already_unlocked = tx_verifier::timelock_check::check_timelock( &block_height, - &median_time, + &block_timestamp, lock, &block_height, &median_time, @@ -1271,7 +1287,7 @@ async fn update_tables_from_transaction_outputs( .await .expect("Unable to set utxo"); } else { - let lock = UtxoLock::from_output_lock(*lock, median_time, block_height); + let lock = UtxoLock::from_output_lock(*lock, block_timestamp, block_height); let utxo = LockedUtxo::new(output.clone(), token_decimals, lock); db_tx .set_locked_utxo_at_height(outpoint, utxo, address.as_str(), block_height) @@ -1360,7 +1376,12 @@ async fn decrease_address_amount( .expect("Unable to get balance") .unwrap_or(Amount::ZERO); - let new_amount = current_balance.sub(*amount).expect("Balance should not overflow"); + let new_amount = current_balance.sub(*amount).unwrap_or_else(|| { + panic!( + "Balance should not overflow {:?} {:?} {:?}", + coin_or_token_id, current_balance, *amount + ) + }); db_tx .set_address_balance_at_height(address.as_str(), new_amount, coin_or_token_id, block_height) @@ -1381,7 +1402,12 @@ async fn decrease_address_locked_amount( .expect("Unable to get balance") .unwrap_or(Amount::ZERO); - let new_amount = current_balance.sub(*amount).expect("Balance should not overflow"); + let new_amount = current_balance.sub(*amount).unwrap_or_else(|| { + panic!( + "Balance should not overflow {:?} {:?} {:?}", + coin_or_token_id, current_balance, *amount + ) + }); db_tx .set_address_locked_balance_at_height( @@ -1395,16 +1421,15 @@ async fn decrease_address_locked_amount( } async fn set_utxo( - outpoint_source_id: OutPointSourceId, - idx: usize, + outpoint: UtxoOutPoint, output: &TxOutput, + token_decimals: Option, db_tx: &mut T, block_height: BlockHeight, spent: bool, chain_config: &ChainConfig, ) { - let outpoint = UtxoOutPoint::new(outpoint_source_id, idx as u32); - let utxo = Utxo::new(output.clone(), None, spent); + let utxo = Utxo::new(output.clone(), token_decimals, spent); if let Some(destination) = get_tx_output_destination(output) { let address = Address::::new(chain_config, destination.clone()) .expect("Unable to encode destination"); diff --git a/api-server/stack-test-suite/tests/v2/address_all_utxos.rs b/api-server/stack-test-suite/tests/v2/address_all_utxos.rs index 09be73c93d..6494ed2d27 100644 --- a/api-server/stack-test-suite/tests/v2/address_all_utxos.rs +++ b/api-server/stack-test-suite/tests/v2/address_all_utxos.rs @@ -118,9 +118,6 @@ async fn multiple_utxos_to_single_address(#[case] seed: Seed) { .build(); let previous_transaction_id = transaction.transaction().get_id(); - let utxo = - UtxoOutPoint::new(OutPointSourceId::Transaction(previous_transaction_id), 0); - alice_utxos.insert(utxo, previous_tx_out.clone()); let mut previous_witness = InputWitness::Standard( StandardInputSignature::produce_uniparty_signature_for_input( @@ -369,13 +366,6 @@ async fn ok(#[case] seed: Seed) { .build(); let mut previous_transaction_id = transaction.transaction().get_id(); - alice_utxos.insert( - UtxoOutPoint::new( - OutPointSourceId::Transaction(transaction.transaction().get_id()), - 0, - ), - previous_tx_out.clone(), - ); let mut previous_witness = InputWitness::Standard( StandardInputSignature::produce_uniparty_signature_for_input( diff --git a/api-server/storage-test-suite/src/basic.rs b/api-server/storage-test-suite/src/basic.rs index 1d822aaf1f..af8e0bf275 100644 --- a/api-server/storage-test-suite/src/basic.rs +++ b/api-server/storage-test-suite/src/basic.rs @@ -602,14 +602,8 @@ where // should return only once the first utxo and also the locked utxo let utxos = db_tx.get_address_all_utxos(bob_address.as_str()).await.unwrap(); - assert_eq!(utxos.len(), 2); - assert_eq!( - utxos.iter().find(|utxo| utxo.0 == outpoint), - Some(&( - outpoint.clone(), - UtxoWithExtraInfo::new(output.clone(), None) - )) - ); + assert_eq!(utxos.len(), 1); + assert_eq!(utxos.iter().find(|utxo| utxo.0 == outpoint), None,); assert_eq!( utxos.iter().find(|utxo| utxo.0 == locked_outpoint), Some(&(locked_outpoint, UtxoWithExtraInfo::new(locked_output, None))) @@ -930,6 +924,7 @@ where let random_nonce2 = AccountNonce::new(rng.gen::()); let random_delegation = Delegation::new( + random_block_height, Destination::PublicKey(pk.clone()), random_pool_id, random_balance, @@ -937,6 +932,7 @@ where ); let random_delegation2 = Delegation::new( + random_block_height2, Destination::PublicKey(pk.clone()), random_pool_id2, random_balance2, @@ -986,6 +982,7 @@ where let random_nonce = AccountNonce::new(rng.gen::()); let random_delegation_new = Delegation::new( + random_block_height, Destination::PublicKey(pk), random_pool_id, random_balance, From 8e80a5edbb8a034e49eb5472d16afbc8a1f837f0 Mon Sep 17 00:00:00 2001 From: Boris Oncev Date: Thu, 16 May 2024 14:37:29 +0200 Subject: [PATCH 2/2] port simulation tests for api server --- .../src/storage/storage_api/mod.rs | 18 +- .../src/sync/{tests.rs => tests/mod.rs} | 2 + .../scanner-lib/src/sync/tests/simulation.rs | 534 ++++++++++++++++++ .../test-framework/src/staking_pools.rs | 1 + common/src/chain/tokens/rpc.rs | 52 +- 5 files changed, 598 insertions(+), 9 deletions(-) rename api-server/scanner-lib/src/sync/{tests.rs => tests/mod.rs} (99%) create mode 100644 api-server/scanner-lib/src/sync/tests/simulation.rs diff --git a/api-server/api-server-common/src/storage/storage_api/mod.rs b/api-server/api-server-common/src/storage/storage_api/mod.rs index 40e151992f..65dd7791e8 100644 --- a/api-server/api-server-common/src/storage/storage_api/mod.rs +++ b/api-server/api-server-common/src/storage/storage_api/mod.rs @@ -20,8 +20,8 @@ use common::{ block::timestamp::BlockTimestamp, timelock::OutputTimeLock, tokens::{ - IsTokenFreezable, IsTokenFrozen, IsTokenUnfreezable, NftIssuance, TokenId, - TokenTotalSupply, + IsTokenFreezable, IsTokenFrozen, IsTokenUnfreezable, NftIssuance, RPCFungibleTokenInfo, + TokenId, TokenTotalSupply, }, AccountNonce, Block, ChainConfig, DelegationId, Destination, PoolId, SignedTransaction, Transaction, TxOutput, UtxoOutPoint, @@ -296,6 +296,20 @@ impl FungibleTokenData { self.authority = authority; self } + + pub fn into_rpc_token_info(self, token_id: TokenId) -> RPCFungibleTokenInfo { + RPCFungibleTokenInfo { + token_id, + token_ticker: self.token_ticker.into(), + number_of_decimals: self.number_of_decimals, + metadata_uri: self.metadata_uri.into(), + circulating_supply: self.circulating_supply, + total_supply: self.total_supply.into(), + is_locked: self.is_locked, + frozen: common::chain::tokens::RPCIsTokenFrozen::new(self.frozen), + authority: self.authority, + } + } } #[derive(Debug, Clone, PartialEq, Eq, Encode, Decode)] diff --git a/api-server/scanner-lib/src/sync/tests.rs b/api-server/scanner-lib/src/sync/tests/mod.rs similarity index 99% rename from api-server/scanner-lib/src/sync/tests.rs rename to api-server/scanner-lib/src/sync/tests/mod.rs index da834a7297..96feaebaa2 100644 --- a/api-server/scanner-lib/src/sync/tests.rs +++ b/api-server/scanner-lib/src/sync/tests/mod.rs @@ -13,6 +13,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod simulation; + use crate::blockchain_state::BlockchainState; use mempool::FeeRate; diff --git a/api-server/scanner-lib/src/sync/tests/simulation.rs b/api-server/scanner-lib/src/sync/tests/simulation.rs new file mode 100644 index 0000000000..2ca8c1c9ed --- /dev/null +++ b/api-server/scanner-lib/src/sync/tests/simulation.rs @@ -0,0 +1,534 @@ +// Copyright (c) 2024 RBB S.r.l +// opensource@mintlayer.org +// SPDX-License-Identifier: MIT +// Licensed under the MIT License; +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://github.com/mintlayer/mintlayer-core/blob/master/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::blockchain_state::BlockchainState; + +use super::*; + +use std::{ + collections::{BTreeMap, BTreeSet}, + num::NonZeroU64, + sync::Arc, +}; + +use api_server_common::storage::{ + impls::in_memory::transactional::TransactionalApiServerInMemoryStorage, + storage_api::{ + ApiServerStorageRead, ApiServerStorageWrite, ApiServerTransactionRw, Transactional, + UtxoLock, + }, +}; + +use chainstate::{BlockSource, ChainstateConfig}; +use chainstate_test_framework::TestFramework; +use common::{ + chain::{ + block::timestamp::BlockTimestamp, + tokens::{make_token_id, NftIssuance, RPCNonFungibleTokenMetadata, RPCTokenInfo, TokenId}, + AccountNonce, AccountType, ConsensusUpgrade, DelegationId, GenBlockId, NetUpgrades, + OutPointSourceId, PoSChainConfigBuilder, PoolId, TxOutput, UtxoOutPoint, + }, + primitives::{Amount, BlockCount, Idable}, +}; +use crypto::{ + key::{KeyKind, PrivateKey}, + vrf::{VRFKeyKind, VRFPrivateKey}, +}; +use pos_accounting::make_delegation_id; +use rstest::rstest; +use test_utils::random::{make_seedable_rng, Seed}; + +#[rstest] +#[trace] +#[case(test_utils::random::Seed::from_entropy(), 20, 50)] +#[tokio::test] +async fn simulation( + #[case] seed: Seed, + #[case] max_blocks: usize, + #[case] max_tx_per_block: usize, +) { + logging::init_logging(); + let mut rng = make_seedable_rng(seed); + + let (vrf_sk, vrf_pk) = VRFPrivateKey::new_from_rng(&mut rng, VRFKeyKind::Schnorrkel); + let (staking_sk, staking_pk) = PrivateKey::new_from_rng(&mut rng, KeyKind::Secp256k1Schnorr); + let (config_builder, genesis_pool_id) = + chainstate_test_framework::create_chain_config_with_default_staking_pool( + &mut rng, staking_pk, vrf_pk, + ); + + let upgrades = vec![( + BlockHeight::new(0), + ConsensusUpgrade::PoS { + initial_difficulty: None, + config: PoSChainConfigBuilder::new_for_unit_test() + .staking_pool_spend_maturity_block_count(BlockCount::new(5)) + .build(), + }, + )]; + let consensus_upgrades = NetUpgrades::initialize(upgrades).expect("valid net-upgrades"); + + let epoch_length = NonZeroU64::new(rng.gen_range(1..10)).unwrap(); + let sealed_epoch_distance_from_tip = rng.gen_range(1..10); + let chain_config = config_builder + .consensus_upgrades(consensus_upgrades) + .max_future_block_time_offset(std::time::Duration::from_secs(1_000_000)) + .epoch_length(epoch_length) + .sealed_epoch_distance_from_tip(sealed_epoch_distance_from_tip) + .build(); + let target_time = chain_config.target_block_spacing(); + let genesis_pool_outpoint = UtxoOutPoint::new(chain_config.genesis_block_id().into(), 1); + + // Initialize original TestFramework + let mut tf = TestFramework::builder(&mut rng) + .with_chain_config(chain_config.clone()) + .with_chainstate_config(ChainstateConfig::new()) + .with_initial_time_since_genesis(target_time.as_secs()) + .with_staking_pools(BTreeMap::from_iter([( + genesis_pool_id, + ( + staking_sk.clone(), + vrf_sk.clone(), + genesis_pool_outpoint.clone(), + ), + )])) + .build(); + + let storage = { + let mut storage = TransactionalApiServerInMemoryStorage::new(&chain_config); + + let mut db_tx = storage.transaction_rw().await.unwrap(); + db_tx.reinitialize_storage(&chain_config).await.unwrap(); + db_tx.commit().await.unwrap(); + + storage + }; + let mut local_state = BlockchainState::new(Arc::new(chain_config.clone()), storage); + local_state.scan_genesis(chain_config.genesis_block().as_ref()).await.unwrap(); + + let num_blocks = rng.gen_range((max_blocks / 2)..max_blocks); + + let mut utxo_outpoints = Vec::new(); + let mut staking_pools: BTreeSet = BTreeSet::new(); + staking_pools.insert(genesis_pool_id); + + let mut delegations = BTreeSet::new(); + let mut token_ids = BTreeSet::new(); + + let mut data_per_block_height = BTreeMap::new(); + data_per_block_height.insert( + BlockHeight::zero(), + ( + utxo_outpoints.clone(), + staking_pools.clone(), + delegations.clone(), + token_ids.clone(), + ), + ); + + let mut tf_internal_staking_pools = BTreeMap::new(); + tf_internal_staking_pools.insert(BlockHeight::zero(), tf.staking_pools.clone()); + + // Generate a random chain + for current_height in 0..num_blocks { + let create_reorg = rng.gen_bool(0.1); + let height_to_continue_from = if create_reorg { + rng.gen_range(0..=current_height) + } else { + current_height + }; + + tf = if create_reorg { + let blocks = if height_to_continue_from > 0 { + tf.chainstate + .get_mainchain_blocks(BlockHeight::new(1), height_to_continue_from) + .unwrap() + } else { + vec![] + }; + let mut new_tf = TestFramework::builder(&mut rng) + .with_chain_config(chain_config.clone()) + .with_initial_time_since_genesis(target_time.as_secs()) + .with_staking_pools(BTreeMap::from_iter([( + genesis_pool_id, + ( + staking_sk.clone(), + vrf_sk.clone(), + genesis_pool_outpoint.clone(), + ), + )])) + .build(); + for block in blocks { + new_tf.progress_time_seconds_since_epoch(target_time.as_secs()); + new_tf.process_block(block.clone(), BlockSource::Local).unwrap(); + } + new_tf.staking_pools = tf_internal_staking_pools + .get(&BlockHeight::new(height_to_continue_from as u64)) + .unwrap() + .clone(); + // new_tf.key_manager = tf.key_manager; + + (utxo_outpoints, staking_pools, delegations, token_ids) = data_per_block_height + .get(&BlockHeight::new(height_to_continue_from as u64)) + .unwrap() + .clone(); + + new_tf + } else { + tf + }; + + let block_height_to_continue_from = BlockHeight::new(height_to_continue_from as u64); + let mut prev_block_hash = tf + .chainstate + .get_block_id_from_height(&block_height_to_continue_from) + .unwrap() + .unwrap(); + + for block_height_idx in 0..=(current_height - height_to_continue_from) { + let block_height = + BlockHeight::new((height_to_continue_from + block_height_idx) as u64); + + let mut block_builder = tf.make_pos_block_builder().with_random_staking_pool(&mut rng); + + for _ in 0..rng.gen_range(10..max_tx_per_block) { + block_builder = block_builder.add_test_transaction(&mut rng); + } + + let block = block_builder.build(); + for tx in block.transactions() { + let new_utxos = (0..tx.inputs().len()).map(|output_index| { + UtxoOutPoint::new( + OutPointSourceId::Transaction(tx.transaction().get_id()), + output_index as u32, + ) + }); + utxo_outpoints.extend(new_utxos); + + let new_pools = tx.outputs().iter().filter_map(|out| match out { + TxOutput::CreateStakePool(pool_id, _) => Some(pool_id), + TxOutput::Burn(_) + | TxOutput::Transfer(_, _) + | TxOutput::LockThenTransfer(_, _, _) + | TxOutput::DataDeposit(_) + | TxOutput::DelegateStaking(_, _) + | TxOutput::CreateDelegationId(_, _) + | TxOutput::IssueFungibleToken(_) + | TxOutput::ProduceBlockFromStake(_, _) + | TxOutput::IssueNft(_, _, _) => None, + }); + staking_pools.extend(new_pools); + + let new_delegations = tx.outputs().iter().filter_map(|out| match out { + TxOutput::CreateDelegationId(_, _) => { + let input0_outpoint = + tx.inputs().iter().find_map(|input| input.utxo_outpoint()).unwrap(); + Some(make_delegation_id(input0_outpoint)) + } + TxOutput::CreateStakePool(_, _) + | TxOutput::Burn(_) + | TxOutput::Transfer(_, _) + | TxOutput::LockThenTransfer(_, _, _) + | TxOutput::DataDeposit(_) + | TxOutput::DelegateStaking(_, _) + | TxOutput::IssueFungibleToken(_) + | TxOutput::ProduceBlockFromStake(_, _) + | TxOutput::IssueNft(_, _, _) => None, + }); + delegations.extend(new_delegations); + + let new_tokens = tx.outputs().iter().filter_map(|out| match out { + TxOutput::IssueNft(_, _, _) | TxOutput::IssueFungibleToken(_) => { + Some(make_token_id(tx.inputs()).unwrap()) + } + TxOutput::CreateStakePool(_, _) + | TxOutput::Burn(_) + | TxOutput::Transfer(_, _) + | TxOutput::LockThenTransfer(_, _, _) + | TxOutput::DataDeposit(_) + | TxOutput::DelegateStaking(_, _) + | TxOutput::CreateDelegationId(_, _) + | TxOutput::ProduceBlockFromStake(_, _) => None, + }); + token_ids.extend(new_tokens); + } + + prev_block_hash = block.get_id().into(); + tf.process_block(block.clone(), BlockSource::Local).unwrap(); + + // save current state + tf_internal_staking_pools.insert(block_height.next_height(), tf.staking_pools.clone()); + data_per_block_height.insert( + block_height.next_height(), + ( + utxo_outpoints.clone(), + staking_pools.clone(), + delegations.clone(), + token_ids.clone(), + ), + ); + + local_state.scan_blocks(block_height, vec![block]).await.unwrap(); + } + + let block_height = BlockHeight::new(current_height as u64); + let median_time = tf.chainstate.calculate_median_time_past(&prev_block_hash).unwrap(); + + check_utxos( + &tf, + &local_state, + &utxo_outpoints, + median_time, + block_height.next_height(), + ) + .await; + + check_staking_pools(&tf, &local_state, &staking_pools).await; + check_delegations(&tf, &local_state, &delegations).await; + check_tokens(&tf, &local_state, &token_ids).await; + } +} + +async fn check_utxos( + tf: &TestFramework, + local_state: &BlockchainState, + utxos: &Vec, + current_median_time: BlockTimestamp, + current_block_height: BlockHeight, +) { + for outpoint in utxos { + check_utxo( + tf, + local_state, + outpoint.clone(), + current_median_time, + current_block_height, + ) + .await; + } +} + +async fn check_utxo( + tf: &TestFramework, + local_state: &BlockchainState, + outpoint: UtxoOutPoint, + current_median_time: BlockTimestamp, + current_block_height: BlockHeight, +) { + let c_utxo = tf.chainstate.utxo(&outpoint).unwrap(); + + // if this is a locked utxo get the unlock time/height + let unlock = c_utxo + .as_ref() + .and_then(|utxo| utxo.output().timelock().zip(utxo.source().blockchain_height().ok())) + .map(|(lock, height)| { + let block_id = tf.block_id(height.into_int()); + let utxo_block_id = block_id.classify(tf.chainstate.get_chain_config()); + let time_of_tx = match utxo_block_id { + GenBlockId::Block(id) => { + tf.chainstate.get_block_header(id).unwrap().unwrap().timestamp() + } + GenBlockId::Genesis(_) => { + tf.chainstate.get_chain_config().genesis_block().timestamp() + } + }; + UtxoLock::from_output_lock(*lock, time_of_tx, height).into_time_and_height() + }); + + let tx = local_state.storage().transaction_ro().await.unwrap(); + + // fetch the locked utxo + let l_utxo = if let Some((unlock_time, unlock_height)) = unlock { + tx.get_locked_utxos_until_now( + unlock_height.unwrap_or(current_block_height), + ( + current_median_time, + unlock_time.unwrap_or(current_median_time), + ), + ) + .await + .unwrap() + .into_iter() + .find_map(|(out, info)| (out == outpoint).then_some(info)) + } else { + None + }; + + // fetch the unlocked utxo + let s_utxo = tx.get_utxo(outpoint).await.unwrap(); + + match (c_utxo, s_utxo) { + (Some(c_utxo), Some(s_utxo)) => { + // if utxo is in chainstate it should not be spent + assert!(!s_utxo.spent()); + // check outputs are the same + assert_eq!(c_utxo.output(), s_utxo.output()); + } + (None, Some(s_utxo)) => { + // if utxo is not found in chainstate but found in scanner it must be spent + assert!(s_utxo.spent()); + // and not in locked utxos + assert_eq!(l_utxo, None); + } + (Some(c_utxo), None) => { + // if utxo is in chainstate but not in unlocked scanner utxos it must be in the locked + // ones + if let Some(l_utxo) = l_utxo { + assert_eq!(c_utxo.output(), &l_utxo.output); + } else { + panic!("Utxo in chainstate but not in the scanner state"); + } + } + (None, None) => { + // on reorg utxos will be gone from both chainstate and the scanner + // same for locked + assert_eq!(l_utxo, None); + } + }; +} + +async fn check_staking_pools( + tf: &TestFramework, + local_state: &BlockchainState, + staking_pools: &BTreeSet, +) { + for pool_id in staking_pools { + check_pool(tf, local_state, *pool_id).await; + } +} + +async fn check_pool( + tf: &TestFramework, + local_state: &BlockchainState, + pool_id: PoolId, +) { + let tx = local_state.storage().transaction_ro().await.unwrap(); + let scanner_data = tx.get_pool_data(pool_id).await.unwrap().unwrap(); + + if let Some(node_data) = tf.chainstate.get_stake_pool_data(pool_id).unwrap() { + // check all fields are the same + assert_eq!(node_data, scanner_data); + } else { + // the pool has been decommissioned + assert_eq!(Amount::ZERO, scanner_data.pledge_amount()); + assert_eq!(Amount::ZERO, scanner_data.staker_balance().unwrap()); + } + + // Compare the delegation shares + let node_delegations = tf + .chainstate + .get_stake_pool_delegations_shares(pool_id) + .unwrap() + .unwrap_or_default(); + + let scanner_delegations = tx.get_pool_delegations(pool_id).await.unwrap(); + + // check all delegations from the node are contained in the scanner + for (id, share) in &node_delegations { + let scanner_delegation = scanner_delegations.get(id).unwrap(); + // check the shares are the same + assert_eq!(share, scanner_delegation.balance()); + } + + // delegations that have not been staked yet are stored in the scanner but not in the node + for (id, scanner_delegation) in scanner_delegations { + let share = node_delegations.get(&id).unwrap_or(&Amount::ZERO); + // check the shares are the same + assert_eq!(share, scanner_delegation.balance()); + } +} + +async fn check_delegations( + tf: &TestFramework, + local_state: &BlockchainState, + delegations: &BTreeSet, +) { + for delegation_id in delegations { + check_delegation(tf, local_state, *delegation_id).await + } +} + +async fn check_delegation( + tf: &TestFramework, + local_state: &BlockchainState, + delegation_id: DelegationId, +) { + let tx = local_state.storage().transaction_ro().await.unwrap(); + let scanner_data = tx.get_delegation(delegation_id).await.unwrap().unwrap(); + + if let Some(node_data) = tf.chainstate.get_stake_delegation_data(delegation_id).unwrap() { + assert_eq!(node_data.source_pool(), scanner_data.pool_id()); + assert_eq!( + node_data.spend_destination(), + scanner_data.spend_destination() + ); + + // check delegation balances are the same + let node_delegation_balance = tf + .chainstate + .get_stake_delegation_balance(delegation_id) + .unwrap() + .unwrap_or(Amount::ZERO); + assert_eq!(node_delegation_balance, *scanner_data.balance()); + + let node_acc_next_nonce = tf + .chainstate + .get_account_nonce_count(AccountType::Delegation(delegation_id)) + .unwrap() + .map_or(AccountNonce::new(0), |nonce| nonce.increment().unwrap()); + assert_eq!(&node_acc_next_nonce, scanner_data.next_nonce()); + } else { + // the pool has been decommissioned + assert_eq!(Amount::ZERO, *scanner_data.balance()); + } +} + +async fn check_tokens( + tf: &TestFramework, + local_state: &BlockchainState, + token_ids: &BTreeSet, +) { + for token_id in token_ids { + check_token(tf, local_state, *token_id).await; + } +} + +async fn check_token( + tf: &TestFramework, + local_state: &BlockchainState, + token_id: TokenId, +) { + let tx = local_state.storage().transaction_ro().await.unwrap(); + let node_data = tf.chainstate.get_token_info_for_rpc(token_id).unwrap().unwrap(); + + match node_data { + RPCTokenInfo::FungibleToken(node_data) => { + let scanner_data = tx.get_fungible_token_issuance(token_id).await.unwrap().unwrap(); + let scanner_data = scanner_data.into_rpc_token_info(token_id); + assert_eq!(node_data, scanner_data); + } + RPCTokenInfo::NonFungibleToken(node_data) => { + let scanner_data = tx.get_nft_token_issuance(token_id).await.unwrap().unwrap(); + + match scanner_data { + NftIssuance::V0(scanner_data) => { + let scanner_metadata: RPCNonFungibleTokenMetadata = + (&scanner_data.metadata).into(); + assert_eq!(node_data.metadata, scanner_metadata); + } + } + } + } +} diff --git a/chainstate/test-framework/src/staking_pools.rs b/chainstate/test-framework/src/staking_pools.rs index 38d764e396..6cbe386abd 100644 --- a/chainstate/test-framework/src/staking_pools.rs +++ b/chainstate/test-framework/src/staking_pools.rs @@ -20,6 +20,7 @@ use common::chain::{PoolId, UtxoOutPoint}; use crypto::{key::PrivateKey, vrf::VRFPrivateKey}; /// Struct that holds possible pools and info required for staking +#[derive(Clone)] pub struct StakingPools { staking_pools: BTreeMap, } diff --git a/common/src/chain/tokens/rpc.rs b/common/src/chain/tokens/rpc.rs index 9277cc984e..c1bf2f384c 100644 --- a/common/src/chain/tokens/rpc.rs +++ b/common/src/chain/tokens/rpc.rs @@ -56,7 +56,16 @@ impl RPCTokenInfo { } #[derive( - Debug, Clone, Copy, Encode, Decode, serde::Serialize, serde::Deserialize, HasValueHint, + Debug, + Clone, + Copy, + PartialEq, + Eq, + Encode, + Decode, + serde::Serialize, + serde::Deserialize, + HasValueHint, )] pub enum RPCTokenTotalSupply { Fixed(Amount), @@ -76,7 +85,16 @@ impl From for RPCTokenTotalSupply { // Indicates whether a token an be frozen #[derive( - Debug, Copy, Clone, Encode, Decode, serde::Serialize, serde::Deserialize, HasValueHint, + Debug, + Copy, + Clone, + PartialEq, + Eq, + Encode, + Decode, + serde::Serialize, + serde::Deserialize, + HasValueHint, )] pub enum RPCIsTokenFreezable { #[codec(index = 0)] @@ -96,7 +114,16 @@ impl From for RPCIsTokenFreezable { // Indicates whether a token an be unfrozen after being frozen #[derive( - Debug, Copy, Clone, Encode, Decode, serde::Serialize, serde::Deserialize, HasValueHint, + Debug, + Copy, + Clone, + PartialEq, + Eq, + Encode, + Decode, + serde::Serialize, + serde::Deserialize, + HasValueHint, )] pub enum RPCIsTokenUnfreezable { #[codec(index = 0)] @@ -118,7 +145,16 @@ impl From for RPCIsTokenUnfreezable { // Meaning transfers, burns, minting, unminting, supply locks etc. Frozen token can only be unfrozen // is such an option was provided while freezing. #[derive( - Debug, Copy, Clone, Encode, Decode, serde::Serialize, serde::Deserialize, HasValueHint, + Debug, + Copy, + Clone, + PartialEq, + Eq, + Encode, + Decode, + serde::Serialize, + serde::Deserialize, + HasValueHint, )] pub enum RPCIsTokenFrozen { #[codec(index = 0)] @@ -136,7 +172,7 @@ impl RPCIsTokenFrozen { } } -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, HasValueHint)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, HasValueHint)] pub struct RPCFungibleTokenInfo { // TODO: Add the controller public key to issuance data - https://github.com/mintlayer/mintlayer-core/issues/401 pub token_id: TokenId, @@ -201,7 +237,9 @@ impl RPCNonFungibleTokenInfo { } } -#[derive(Debug, Clone, Encode, Decode, serde::Serialize, serde::Deserialize, HasValueHint)] +#[derive( + Debug, Clone, PartialEq, Eq, Encode, Decode, serde::Serialize, serde::Deserialize, HasValueHint, +)] pub struct RPCTokenCreator(Vec); impl From<&TokenCreator> for RPCTokenCreator { @@ -211,7 +249,7 @@ impl From<&TokenCreator> for RPCTokenCreator { } } -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, HasValueHint)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, HasValueHint)] pub struct RPCNonFungibleTokenMetadata { pub creator: Option, pub name: RpcString,