From 468881fb2ecbe6ab1344079ef3a7abe31bbf134c Mon Sep 17 00:00:00 2001 From: ananas-block Date: Tue, 24 Jun 2025 23:29:03 +0100 Subject: [PATCH 01/21] chore: add test setup --- src/ingester/persist/mod.rs | 2 + tests/integration_tests/main.rs | 1 + .../persist_state_update_test.rs | 66 +++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 tests/integration_tests/persist_state_update_test.rs diff --git a/src/ingester/persist/mod.rs b/src/ingester/persist/mod.rs index 57734af6..16aee808 100644 --- a/src/ingester/persist/mod.rs +++ b/src/ingester/persist/mod.rs @@ -65,7 +65,9 @@ pub async fn persist_state_update( return Ok(()); } let StateUpdate { + // input accounts that will be nullified in_accounts, + // Output accounts that will be created out_accounts, account_transactions, transactions, diff --git a/tests/integration_tests/main.rs b/tests/integration_tests/main.rs index 325e77e0..f4dded42 100644 --- a/tests/integration_tests/main.rs +++ b/tests/integration_tests/main.rs @@ -7,6 +7,7 @@ mod batched_state_tree_tests; mod e2e_tests; mod mock_tests; mod open_api_tests; +mod persist_state_update_test; mod prod_tests; mod snapshot_tests; mod utils; diff --git a/tests/integration_tests/persist_state_update_test.rs b/tests/integration_tests/persist_state_update_test.rs new file mode 100644 index 00000000..082eb62d --- /dev/null +++ b/tests/integration_tests/persist_state_update_test.rs @@ -0,0 +1,66 @@ +use crate::utils::*; +use function_name::named; +use photon_indexer::ingester::parser::state_update::StateUpdate; +use photon_indexer::ingester::persist::persist_state_update; +use sea_orm::{DatabaseConnection, EntityTrait, PaginatorTrait, TransactionTrait}; +use serial_test::serial; +use std::env; + +/// Helper function to persist a state update and commit the transaction +async fn persist_state_update_and_commit( + db_conn: &DatabaseConnection, + state_update: StateUpdate, +) -> Result<(), Box> { + let txn = db_conn.begin().await?; + persist_state_update(&txn, state_update).await?; + txn.commit().await?; + Ok(()) +} + +#[named] +#[rstest] +#[tokio::test] +#[serial] +async fn test_persist_empty_state_update( + #[values(DatabaseBackend::Sqlite)] db_backend: DatabaseBackend, +) { + // Set required environment variables + env::set_var("MAINNET_RPC_URL", "https://api.mainnet-beta.solana.com"); + env::set_var("DEVNET_RPC_URL", "https://api.devnet.solana.com"); + + let name = trim_test_name(function_name!()); + let setup = setup(name, db_backend).await; + + // Create an empty state update + let empty_state_update = StateUpdate::default(); + + // Call persist_state_update with empty state update and commit + let result = persist_state_update_and_commit(&setup.db_conn, empty_state_update).await; + + // Should complete successfully + assert!(result.is_ok()); + + // Verify that key tables remain empty after persisting empty state update + use photon_indexer::dao::generated::{account_transactions, accounts, transactions}; + + let accounts_count = accounts::Entity::find() + .count(setup.db_conn.as_ref()) + .await + .unwrap(); + assert_eq!(accounts_count, 0, "Accounts table should be empty"); + + let transactions_count = transactions::Entity::find() + .count(setup.db_conn.as_ref()) + .await + .unwrap(); + assert_eq!(transactions_count, 0, "Transactions table should be empty"); + + let account_transactions_count = account_transactions::Entity::find() + .count(setup.db_conn.as_ref()) + .await + .unwrap(); + assert_eq!( + account_transactions_count, 0, + "Account transactions table should be empty" + ); +} From 5e293903d1a6bf1ce154b8c79805209149ba4472 Mon Sep 17 00:00:00 2001 From: ananas-block Date: Tue, 24 Jun 2025 23:45:03 +0100 Subject: [PATCH 02/21] stash rnd stateupdate --- .../persist_state_update_test.rs | 262 +++++++++++++++++- 1 file changed, 261 insertions(+), 1 deletion(-) diff --git a/tests/integration_tests/persist_state_update_test.rs b/tests/integration_tests/persist_state_update_test.rs index 082eb62d..7dd6f5cd 100644 --- a/tests/integration_tests/persist_state_update_test.rs +++ b/tests/integration_tests/persist_state_update_test.rs @@ -1,11 +1,229 @@ use crate::utils::*; use function_name::named; -use photon_indexer::ingester::parser::state_update::StateUpdate; +use photon_indexer::common::typedefs::account::{Account, AccountContext, AccountWithContext}; +use photon_indexer::common::typedefs::account::AccountData; +use photon_indexer::common::typedefs::bs64_string::Base64String; +use photon_indexer::common::typedefs::hash::Hash; +use photon_indexer::common::typedefs::serializable_pubkey::SerializablePubkey; +use photon_indexer::common::typedefs::unsigned_integer::UnsignedInteger; +use photon_indexer::ingester::parser::indexer_events::RawIndexedElement; +use photon_indexer::ingester::parser::state_update::{ + AccountTransaction, AddressQueueUpdate, IndexedTreeLeafUpdate, LeafNullification, StateUpdate, Transaction +}; use photon_indexer::ingester::persist::persist_state_update; +use light_compressed_account::indexer_event::event::BatchNullifyContext; +use rand::{rngs::{StdRng, ThreadRng}, Rng, RngCore, SeedableRng}; use sea_orm::{DatabaseConnection, EntityTrait, PaginatorTrait, TransactionTrait}; use serial_test::serial; +use solana_pubkey::Pubkey; +use solana_sdk::signature::Signature; use std::env; +/// Configuration for generating random collections +#[derive(Debug, Clone)] +pub struct CollectionConfig { + pub min_entries: usize, + pub max_entries: usize, + pub probability: f64, +} + +impl CollectionConfig { + pub fn new(min_entries: usize, max_entries: usize, probability: f64) -> Self { + Self { + min_entries, + max_entries, + probability, + } + } +} + +/// Configuration for generating random StateUpdate data +#[derive(Debug, Clone)] +pub struct StateUpdateConfig { + // Collection configurations for StateUpdate fields + pub in_accounts: CollectionConfig, + pub out_accounts: CollectionConfig, + pub account_transactions: CollectionConfig, + pub transactions: CollectionConfig, + pub leaf_nullifications: CollectionConfig, + pub indexed_merkle_tree_updates: CollectionConfig, + pub batch_nullify_context: CollectionConfig, + pub batch_new_addresses: CollectionConfig, + + // Value ranges for various types + pub lamports_min: u64, + pub lamports_max: u64, + pub slot_min: u64, + pub slot_max: u64, + pub seq_min: u64, + pub seq_max: u64, + pub leaf_index_min: u64, + pub leaf_index_max: u64, + pub data_size_min: usize, + pub data_size_max: usize, +} + +impl Default for StateUpdateConfig { + fn default() -> Self { + Self { + in_accounts: CollectionConfig::new(0, 5, 0.7), + out_accounts: CollectionConfig::new(0, 5, 0.8), + account_transactions: CollectionConfig::new(0, 3, 0.6), + transactions: CollectionConfig::new(0, 2, 0.9), + leaf_nullifications: CollectionConfig::new(0, 3, 0.5), + indexed_merkle_tree_updates: CollectionConfig::new(0, 3, 0.4), + batch_nullify_context: CollectionConfig::new(0, 2, 0.3), + batch_new_addresses: CollectionConfig::new(0, 3, 0.6), + + lamports_min: 1000, + lamports_max: 1_000_000, + slot_min: 0, + slot_max: 10_000, + seq_min: 0, + seq_max: 10_000, + leaf_index_min: 0, + leaf_index_max: 100, + data_size_min: 0, + data_size_max: 100, + } + } +} + +/// Generate a random StateUpdate following the light-protocol pattern +fn get_rnd_state_update(rng: &mut StdRng, config: &StateUpdateConfig) -> StateUpdate { + let mut state_update = StateUpdate::default(); + + // Generate in_accounts (HashSet) + if rng.gen_bool(config.in_accounts.probability) { + let count = rng.gen_range(config.in_accounts.min_entries..=config.in_accounts.max_entries); + for _ in 0..count { + state_update.in_accounts.insert(Hash::new_unique()); + } + } + + // Generate out_accounts (Vec) + if rng.gen_bool(config.out_accounts.probability) { + let count = rng.gen_range(config.out_accounts.min_entries..=config.out_accounts.max_entries); + for _ in 0..count { + let account = AccountWithContext { + account: Account { + hash: Hash::new_unique(), + address: if rng.gen_bool(0.7) { Some(SerializablePubkey::new_unique()) } else { None }, + data: if rng.gen_bool(0.6) { + let data_size = rng.gen_range(config.data_size_min..=config.data_size_max); + Some(AccountData { + discriminator: UnsignedInteger(rng.gen()), + data: Base64String((0..data_size).map(|_| rng.gen()).collect()), + data_hash: Hash::new_unique(), + }) + } else { None }, + owner: SerializablePubkey::new_unique(), + lamports: UnsignedInteger(rng.gen_range(config.lamports_min..=config.lamports_max)), + tree: SerializablePubkey::new_unique(), + leaf_index: UnsignedInteger(rng.gen_range(config.leaf_index_min..=config.leaf_index_max)), + seq: Some(UnsignedInteger(rng.gen_range(config.seq_min..=config.seq_max))), + slot_created: UnsignedInteger(rng.gen_range(config.slot_min..=config.slot_max)), + }, + context: AccountContext::default(), + }; + state_update.out_accounts.push(account); + } + } + + // Generate account_transactions (HashSet) + if rng.gen_bool(config.account_transactions.probability) { + let count = rng.gen_range(config.account_transactions.min_entries..=config.account_transactions.max_entries); + for _ in 0..count { + let mut sig_bytes = [0u8; 64]; + rng.fill(&mut sig_bytes); + state_update.account_transactions.insert(AccountTransaction { + hash: Hash::new_unique(), + signature: Signature::from(sig_bytes), + }); + } + } + + // Generate transactions (HashSet) + if rng.gen_bool(config.transactions.probability) { + let count = rng.gen_range(config.transactions.min_entries..=config.transactions.max_entries); + for _ in 0..count { + let mut sig_bytes = [0u8; 64]; + rng.fill(&mut sig_bytes); + state_update.transactions.insert(Transaction { + signature: Signature::from(sig_bytes), + slot: rng.gen_range(config.slot_min..=config.slot_max), + uses_compression: rng.gen(), + error: if rng.gen_bool(0.1) { Some("Random error".to_string()) } else { None }, + }); + } + } + + // Generate leaf_nullifications (HashSet) + if rng.gen_bool(config.leaf_nullifications.probability) { + let count = rng.gen_range(config.leaf_nullifications.min_entries..=config.leaf_nullifications.max_entries); + for _ in 0..count { + let mut sig_bytes = [0u8; 64]; + rng.fill(&mut sig_bytes); + state_update.leaf_nullifications.insert(LeafNullification { + tree: Pubkey::new_unique(), + leaf_index: rng.gen_range(config.leaf_index_min..=config.leaf_index_max), + seq: rng.gen_range(config.seq_min..=config.seq_max), + signature: Signature::from(sig_bytes), + }); + } + } + + // Generate indexed_merkle_tree_updates (HashMap<(Pubkey, u64), IndexedTreeLeafUpdate>) + if rng.gen_bool(config.indexed_merkle_tree_updates.probability) { + let count = rng.gen_range(config.indexed_merkle_tree_updates.min_entries..=config.indexed_merkle_tree_updates.max_entries); + for _ in 0..count { + let tree = Pubkey::new_unique(); + let index = rng.gen_range(config.leaf_index_min..=config.leaf_index_max); + let update = IndexedTreeLeafUpdate { + tree, + leaf: RawIndexedElement { + value: rng.gen::<[u8; 32]>(), + next_index: rng.gen::() as usize, + next_value: rng.gen::<[u8; 32]>(), + index: rng.gen::() as usize, + }, + hash: rng.gen::<[u8; 32]>(), + seq: rng.gen_range(config.seq_min..=config.seq_max), + }; + state_update.indexed_merkle_tree_updates.insert((tree, index), update); + } + } + + // Generate batch_nullify_context (Vec) + if rng.gen_bool(config.batch_nullify_context.probability) { + let count = rng.gen_range(config.batch_nullify_context.min_entries..=config.batch_nullify_context.max_entries); + for _ in 0..count { + state_update.batch_nullify_context.push(BatchNullifyContext { + tx_hash: rng.gen::<[u8; 32]>(), + account_hash: rng.gen::<[u8; 32]>(), + nullifier: rng.gen::<[u8; 32]>(), + nullifier_queue_index: rng.gen_range(0..=1000), + }); + } + } + + // Generate batch_new_addresses (Vec) + if rng.gen_bool(config.batch_new_addresses.probability) { + let count = rng.gen_range(config.batch_new_addresses.min_entries..=config.batch_new_addresses.max_entries); + for _ in 0..count { + state_update.batch_new_addresses.push(AddressQueueUpdate { + tree: SerializablePubkey::new_unique(), + address: rng.gen::<[u8; 32]>(), + queue_index: rng.gen_range(0..=1000), + }); + } + } + + // Note: batch_merkle_tree_events is left as default since it's complex and rarely used + + state_update +} + /// Helper function to persist a state update and commit the transaction async fn persist_state_update_and_commit( db_conn: &DatabaseConnection, @@ -28,6 +246,15 @@ async fn test_persist_empty_state_update( env::set_var("MAINNET_RPC_URL", "https://api.mainnet-beta.solana.com"); env::set_var("DEVNET_RPC_URL", "https://api.devnet.solana.com"); + // Set up deterministic randomness following the light-protocol pattern + let mut thread_rng = ThreadRng::default(); + let random_seed = thread_rng.next_u64(); + let seed: u64 = random_seed; // Could optionally take seed as parameter + // Keep this print so that in case the test fails + // we can use the seed to reproduce the error. + println!("\n\npersist_state_update test seed {}\n\n", seed); + let mut _rng = StdRng::seed_from_u64(seed); + let name = trim_test_name(function_name!()); let setup = setup(name, db_backend).await; @@ -64,3 +291,36 @@ async fn test_persist_empty_state_update( "Account transactions table should be empty" ); } + +#[named] +#[rstest] +#[tokio::test] +#[serial] +async fn test_config_structure( + #[values(DatabaseBackend::Sqlite)] db_backend: DatabaseBackend, +) { + // Set required environment variables + env::set_var("MAINNET_RPC_URL", "https://api.mainnet-beta.solana.com"); + env::set_var("DEVNET_RPC_URL", "https://api.devnet.solana.com"); + + let name = trim_test_name(function_name!()); + let setup = setup(name, db_backend).await; + + // Test that the new config structure works correctly + let config = StateUpdateConfig::default(); + + // Verify config structure values + assert_eq!(config.in_accounts.min_entries, 0); + assert_eq!(config.in_accounts.max_entries, 5); + assert_eq!(config.in_accounts.probability, 0.7); + + assert_eq!(config.out_accounts.min_entries, 0); + assert_eq!(config.out_accounts.max_entries, 5); + assert_eq!(config.out_accounts.probability, 0.8); + + assert_eq!(config.transactions.min_entries, 0); + assert_eq!(config.transactions.max_entries, 2); + assert_eq!(config.transactions.probability, 0.9); + + println!("Config structure test completed successfully - unified CollectionConfig approach working"); +} From 4d5c4c5c9339c1bee93cd23bd3e202d9b4b99508 Mon Sep 17 00:00:00 2001 From: ananas-block Date: Wed, 25 Jun 2025 00:28:35 +0100 Subject: [PATCH 03/21] add assert_output_accounts_persisted test works --- .../persist_state_update_test.rs | 364 +++++++++++++----- 1 file changed, 264 insertions(+), 100 deletions(-) diff --git a/tests/integration_tests/persist_state_update_test.rs b/tests/integration_tests/persist_state_update_test.rs index 7dd6f5cd..82679ac0 100644 --- a/tests/integration_tests/persist_state_update_test.rs +++ b/tests/integration_tests/persist_state_update_test.rs @@ -1,24 +1,34 @@ use crate::utils::*; use function_name::named; -use photon_indexer::common::typedefs::account::{Account, AccountContext, AccountWithContext}; +use light_compressed_account::indexer_event::event::BatchNullifyContext; use photon_indexer::common::typedefs::account::AccountData; +use photon_indexer::common::typedefs::account::{Account, AccountContext, AccountWithContext}; use photon_indexer::common::typedefs::bs64_string::Base64String; use photon_indexer::common::typedefs::hash::Hash; use photon_indexer::common::typedefs::serializable_pubkey::SerializablePubkey; use photon_indexer::common::typedefs::unsigned_integer::UnsignedInteger; use photon_indexer::ingester::parser::indexer_events::RawIndexedElement; use photon_indexer::ingester::parser::state_update::{ - AccountTransaction, AddressQueueUpdate, IndexedTreeLeafUpdate, LeafNullification, StateUpdate, Transaction + AccountTransaction, AddressQueueUpdate, IndexedTreeLeafUpdate, LeafNullification, StateUpdate, + Transaction, }; +use photon_indexer::ingester::parser::tree_info::TreeInfo; use photon_indexer::ingester::persist::persist_state_update; -use light_compressed_account::indexer_event::event::BatchNullifyContext; -use rand::{rngs::{StdRng, ThreadRng}, Rng, RngCore, SeedableRng}; -use sea_orm::{DatabaseConnection, EntityTrait, PaginatorTrait, TransactionTrait}; +use rand::{ + rngs::{StdRng, ThreadRng}, + Rng, RngCore, SeedableRng, +}; +use sea_orm::{ + prelude::Decimal, DatabaseConnection, EntityTrait, PaginatorTrait, QueryFilter, + TransactionTrait, +}; use serial_test::serial; -use solana_pubkey::Pubkey; use solana_sdk::signature::Signature; use std::env; +// Use the specific tree from QUEUE_TREE_MAPPING +const TEST_TREE_PUBKEY_STR: &str = "smt1NamzXdq4AMqS2fS2F1i5KTYPZRhoHgWx38d8WsT"; + /// Configuration for generating random collections #[derive(Debug, Clone)] pub struct CollectionConfig { @@ -49,16 +59,12 @@ pub struct StateUpdateConfig { pub indexed_merkle_tree_updates: CollectionConfig, pub batch_nullify_context: CollectionConfig, pub batch_new_addresses: CollectionConfig, - + // Value ranges for various types pub lamports_min: u64, pub lamports_max: u64, - pub slot_min: u64, - pub slot_max: u64, - pub seq_min: u64, - pub seq_max: u64, - pub leaf_index_min: u64, - pub leaf_index_max: u64, + pub discriminator_min: u64, + pub discriminator_max: u64, pub data_size_min: usize, pub data_size_max: usize, } @@ -66,23 +72,19 @@ pub struct StateUpdateConfig { impl Default for StateUpdateConfig { fn default() -> Self { Self { - in_accounts: CollectionConfig::new(0, 5, 0.7), - out_accounts: CollectionConfig::new(0, 5, 0.8), - account_transactions: CollectionConfig::new(0, 3, 0.6), - transactions: CollectionConfig::new(0, 2, 0.9), - leaf_nullifications: CollectionConfig::new(0, 3, 0.5), - indexed_merkle_tree_updates: CollectionConfig::new(0, 3, 0.4), - batch_nullify_context: CollectionConfig::new(0, 2, 0.3), - batch_new_addresses: CollectionConfig::new(0, 3, 0.6), - + in_accounts: CollectionConfig::new(0, 5, 0.0), + out_accounts: CollectionConfig::new(0, 5, 1.0), + account_transactions: CollectionConfig::new(0, 3, 0.0), + transactions: CollectionConfig::new(0, 2, 0.0), + leaf_nullifications: CollectionConfig::new(0, 3, 0.0), + indexed_merkle_tree_updates: CollectionConfig::new(0, 3, 0.0), + batch_nullify_context: CollectionConfig::new(0, 2, 0.0), + batch_new_addresses: CollectionConfig::new(0, 3, 0.0), + lamports_min: 1000, lamports_max: 1_000_000, - slot_min: 0, - slot_max: 10_000, - seq_min: 0, - seq_max: 10_000, - leaf_index_min: 0, - leaf_index_max: 100, + discriminator_min: 1000, + discriminator_max: 1_000_000, data_size_min: 0, data_size_max: 100, } @@ -90,137 +92,182 @@ impl Default for StateUpdateConfig { } /// Generate a random StateUpdate following the light-protocol pattern -fn get_rnd_state_update(rng: &mut StdRng, config: &StateUpdateConfig) -> StateUpdate { +fn get_rnd_state_update( + rng: &mut StdRng, + config: &StateUpdateConfig, + slot: u64, + base_seq: u64, + base_leaf_index: u64, +) -> StateUpdate { let mut state_update = StateUpdate::default(); - + + // Get tree info from QUEUE_TREE_MAPPING + let tree_info = + TreeInfo::get(TEST_TREE_PUBKEY_STR).expect("Test tree should exist in QUEUE_TREE_MAPPING"); + let test_tree_pubkey = tree_info.tree; + // Generate in_accounts (HashSet) if rng.gen_bool(config.in_accounts.probability) { let count = rng.gen_range(config.in_accounts.min_entries..=config.in_accounts.max_entries); - for _ in 0..count { + for _i in 0..count { state_update.in_accounts.insert(Hash::new_unique()); } } - + // Generate out_accounts (Vec) if rng.gen_bool(config.out_accounts.probability) { - let count = rng.gen_range(config.out_accounts.min_entries..=config.out_accounts.max_entries); - for _ in 0..count { + let count = + rng.gen_range(config.out_accounts.min_entries..=config.out_accounts.max_entries); + for i in 0..count { let account = AccountWithContext { account: Account { hash: Hash::new_unique(), - address: if rng.gen_bool(0.7) { Some(SerializablePubkey::new_unique()) } else { None }, + address: if rng.gen_bool(0.7) { + Some(SerializablePubkey::new_unique()) + } else { + None + }, data: if rng.gen_bool(0.6) { let data_size = rng.gen_range(config.data_size_min..=config.data_size_max); Some(AccountData { - discriminator: UnsignedInteger(rng.gen()), + discriminator: UnsignedInteger( + rng.gen_range(config.discriminator_min..=config.discriminator_max), + ), data: Base64String((0..data_size).map(|_| rng.gen()).collect()), data_hash: Hash::new_unique(), }) - } else { None }, + } else { + None + }, owner: SerializablePubkey::new_unique(), - lamports: UnsignedInteger(rng.gen_range(config.lamports_min..=config.lamports_max)), - tree: SerializablePubkey::new_unique(), - leaf_index: UnsignedInteger(rng.gen_range(config.leaf_index_min..=config.leaf_index_max)), - seq: Some(UnsignedInteger(rng.gen_range(config.seq_min..=config.seq_max))), - slot_created: UnsignedInteger(rng.gen_range(config.slot_min..=config.slot_max)), + lamports: UnsignedInteger( + rng.gen_range(config.lamports_min as i64..=config.lamports_max as i64) + as u64, + ), + tree: SerializablePubkey::from(test_tree_pubkey), + leaf_index: UnsignedInteger(base_leaf_index + i as u64), + seq: Some(UnsignedInteger(base_seq + i as u64)), + slot_created: UnsignedInteger(slot), }, context: AccountContext::default(), }; state_update.out_accounts.push(account); } } - + // Generate account_transactions (HashSet) if rng.gen_bool(config.account_transactions.probability) { - let count = rng.gen_range(config.account_transactions.min_entries..=config.account_transactions.max_entries); - for _ in 0..count { + let count = rng.gen_range( + config.account_transactions.min_entries..=config.account_transactions.max_entries, + ); + for _i in 0..count { let mut sig_bytes = [0u8; 64]; rng.fill(&mut sig_bytes); - state_update.account_transactions.insert(AccountTransaction { - hash: Hash::new_unique(), - signature: Signature::from(sig_bytes), - }); + state_update + .account_transactions + .insert(AccountTransaction { + hash: Hash::new_unique(), + signature: Signature::from(sig_bytes), + }); } } - + // Generate transactions (HashSet) if rng.gen_bool(config.transactions.probability) { - let count = rng.gen_range(config.transactions.min_entries..=config.transactions.max_entries); - for _ in 0..count { + let count = + rng.gen_range(config.transactions.min_entries..=config.transactions.max_entries); + for _i in 0..count { let mut sig_bytes = [0u8; 64]; rng.fill(&mut sig_bytes); state_update.transactions.insert(Transaction { signature: Signature::from(sig_bytes), - slot: rng.gen_range(config.slot_min..=config.slot_max), + slot, uses_compression: rng.gen(), - error: if rng.gen_bool(0.1) { Some("Random error".to_string()) } else { None }, + error: if rng.gen_bool(0.1) { + Some("Random error".to_string()) + } else { + None + }, }); } } - + // Generate leaf_nullifications (HashSet) if rng.gen_bool(config.leaf_nullifications.probability) { - let count = rng.gen_range(config.leaf_nullifications.min_entries..=config.leaf_nullifications.max_entries); - for _ in 0..count { + let count = rng.gen_range( + config.leaf_nullifications.min_entries..=config.leaf_nullifications.max_entries, + ); + for i in 0..count { let mut sig_bytes = [0u8; 64]; rng.fill(&mut sig_bytes); state_update.leaf_nullifications.insert(LeafNullification { - tree: Pubkey::new_unique(), - leaf_index: rng.gen_range(config.leaf_index_min..=config.leaf_index_max), - seq: rng.gen_range(config.seq_min..=config.seq_max), + tree: test_tree_pubkey, + leaf_index: base_leaf_index + i as u64, + seq: base_seq + i as u64, signature: Signature::from(sig_bytes), }); } } - + // Generate indexed_merkle_tree_updates (HashMap<(Pubkey, u64), IndexedTreeLeafUpdate>) if rng.gen_bool(config.indexed_merkle_tree_updates.probability) { - let count = rng.gen_range(config.indexed_merkle_tree_updates.min_entries..=config.indexed_merkle_tree_updates.max_entries); - for _ in 0..count { - let tree = Pubkey::new_unique(); - let index = rng.gen_range(config.leaf_index_min..=config.leaf_index_max); + let count = rng.gen_range( + config.indexed_merkle_tree_updates.min_entries + ..=config.indexed_merkle_tree_updates.max_entries, + ); + for i in 0..count { + let tree = test_tree_pubkey; + let index = base_leaf_index + i as u64; let update = IndexedTreeLeafUpdate { tree, leaf: RawIndexedElement { value: rng.gen::<[u8; 32]>(), next_index: rng.gen::() as usize, next_value: rng.gen::<[u8; 32]>(), - index: rng.gen::() as usize, + index: (base_leaf_index + i as u64) as usize, }, hash: rng.gen::<[u8; 32]>(), - seq: rng.gen_range(config.seq_min..=config.seq_max), + seq: base_seq + i as u64, }; - state_update.indexed_merkle_tree_updates.insert((tree, index), update); + state_update + .indexed_merkle_tree_updates + .insert((tree, index), update); } } - + // Generate batch_nullify_context (Vec) if rng.gen_bool(config.batch_nullify_context.probability) { - let count = rng.gen_range(config.batch_nullify_context.min_entries..=config.batch_nullify_context.max_entries); - for _ in 0..count { - state_update.batch_nullify_context.push(BatchNullifyContext { - tx_hash: rng.gen::<[u8; 32]>(), - account_hash: rng.gen::<[u8; 32]>(), - nullifier: rng.gen::<[u8; 32]>(), - nullifier_queue_index: rng.gen_range(0..=1000), - }); + let count = rng.gen_range( + config.batch_nullify_context.min_entries..=config.batch_nullify_context.max_entries, + ); + for i in 0..count { + state_update + .batch_nullify_context + .push(BatchNullifyContext { + tx_hash: rng.gen::<[u8; 32]>(), + account_hash: rng.gen::<[u8; 32]>(), + nullifier: rng.gen::<[u8; 32]>(), + nullifier_queue_index: i as u64, + }); } } - + // Generate batch_new_addresses (Vec) if rng.gen_bool(config.batch_new_addresses.probability) { - let count = rng.gen_range(config.batch_new_addresses.min_entries..=config.batch_new_addresses.max_entries); - for _ in 0..count { + let count = rng.gen_range( + config.batch_new_addresses.min_entries..=config.batch_new_addresses.max_entries, + ); + for i in 0..count { state_update.batch_new_addresses.push(AddressQueueUpdate { - tree: SerializablePubkey::new_unique(), + tree: SerializablePubkey::from(test_tree_pubkey), address: rng.gen::<[u8; 32]>(), - queue_index: rng.gen_range(0..=1000), + queue_index: i as u64, }); } } - + // Note: batch_merkle_tree_events is left as default since it's complex and rarely used - + state_update } @@ -235,6 +282,95 @@ async fn persist_state_update_and_commit( Ok(()) } +/// Assert that all output accounts from the state update were inserted correctly into the database +async fn assert_output_accounts_persisted( + db_conn: &DatabaseConnection, + state_update: &StateUpdate, +) -> Result<(), Box> { + use photon_indexer::dao::generated::accounts; + use sea_orm::ColumnTrait; + + if state_update.out_accounts.is_empty() { + // If no accounts expected, verify table is empty + let account_count = accounts::Entity::find().count(db_conn).await?; + assert_eq!( + account_count, 0, + "Expected no accounts in database, but found {}", + account_count + ); + return Ok(()); + } + + // Create expected models from state update + let expected_models: Vec = state_update + .out_accounts + .iter() + .map(|account_with_context| { + let account = &account_with_context.account; + let context = &account_with_context.context; + accounts::Model { + hash: account.hash.0.to_vec(), + data: account.data.as_ref().map(|data| data.data.0.clone()), + data_hash: account.data.as_ref().map(|data| data.data_hash.0.to_vec()), + address: account + .address + .as_ref() + .map(|addr| addr.0.to_bytes().to_vec()), + owner: account.owner.0.to_bytes().to_vec(), + tree: account.tree.0.to_bytes().to_vec(), + leaf_index: account.leaf_index.0 as i64, + seq: account.seq.as_ref().map(|seq| seq.0 as i64), + slot_created: account.slot_created.0 as i64, + spent: false, // Default value for new accounts (from persist logic) + prev_spent: None, // Default value + lamports: Decimal::from(account.lamports.0), + discriminator: account + .data + .as_ref() + .map(|data| Decimal::from(data.discriminator.0)), + tree_type: Some(context.tree_type as i32), // From account context + nullified_in_tree: false, // Default value for new accounts (from persist logic) + nullifier_queue_index: None, // Default value + in_output_queue: context.in_output_queue, // From account context + queue: context.queue.0.to_bytes().to_vec(), // Use queue from account context + nullifier: None, // Default value + tx_hash: None, // Default value + } + }) + .collect(); + + // Get all account hashes for the query + let expected_hashes: Vec> = expected_models + .iter() + .map(|model| model.hash.clone()) + .collect(); + + // Query database for accounts with matching hashes + let mut db_accounts = accounts::Entity::find() + .filter(accounts::Column::Hash.is_in(expected_hashes)) + .all(db_conn) + .await?; + + // Sort both vectors by hash for consistent comparison + let mut expected_models_sorted = expected_models; + expected_models_sorted.sort_by(|a, b| a.hash.cmp(&b.hash)); + db_accounts.sort_by(|a, b| a.hash.cmp(&b.hash)); + + // Single assert comparing the entire vectors + assert_eq!( + db_accounts, expected_models_sorted, + "Database accounts do not match expected accounts" + ); + + println!( + "✅ Successfully verified {} output accounts were persisted correctly", + db_accounts.len() + ); + println!("Database accounts: {:?}", db_accounts); + println!("Expected accounts: {:?}", expected_models_sorted); + Ok(()) +} + #[named] #[rstest] #[tokio::test] @@ -245,16 +381,16 @@ async fn test_persist_empty_state_update( // Set required environment variables env::set_var("MAINNET_RPC_URL", "https://api.mainnet-beta.solana.com"); env::set_var("DEVNET_RPC_URL", "https://api.devnet.solana.com"); - + // Set up deterministic randomness following the light-protocol pattern let mut thread_rng = ThreadRng::default(); let random_seed = thread_rng.next_u64(); let seed: u64 = random_seed; // Could optionally take seed as parameter - // Keep this print so that in case the test fails - // we can use the seed to reproduce the error. + // Keep this print so that in case the test fails + // we can use the seed to reproduce the error. println!("\n\npersist_state_update test seed {}\n\n", seed); let mut _rng = StdRng::seed_from_u64(seed); - + let name = trim_test_name(function_name!()); let setup = setup(name, db_backend).await; @@ -296,31 +432,59 @@ async fn test_persist_empty_state_update( #[rstest] #[tokio::test] #[serial] -async fn test_config_structure( - #[values(DatabaseBackend::Sqlite)] db_backend: DatabaseBackend, -) { +async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: DatabaseBackend) { // Set required environment variables env::set_var("MAINNET_RPC_URL", "https://api.mainnet-beta.solana.com"); env::set_var("DEVNET_RPC_URL", "https://api.devnet.solana.com"); - + let name = trim_test_name(function_name!()); let setup = setup(name, db_backend).await; + // Set up deterministic randomness following the light-protocol pattern + let mut thread_rng = ThreadRng::default(); + let random_seed = thread_rng.next_u64(); + let seed: u64 = random_seed; + println!("\n\nconfig structure test seed {}\n\n", seed); + let mut rng = StdRng::seed_from_u64(seed); + // Test that the new config structure works correctly let config = StateUpdateConfig::default(); - + // Verify config structure values assert_eq!(config.in_accounts.min_entries, 0); assert_eq!(config.in_accounts.max_entries, 5); - assert_eq!(config.in_accounts.probability, 0.7); - + assert_eq!(config.in_accounts.probability, 0.0); + assert_eq!(config.out_accounts.min_entries, 0); assert_eq!(config.out_accounts.max_entries, 5); - assert_eq!(config.out_accounts.probability, 0.8); - + assert_eq!(config.out_accounts.probability, 1.0); + assert_eq!(config.transactions.min_entries, 0); assert_eq!(config.transactions.max_entries, 2); - assert_eq!(config.transactions.probability, 0.9); - - println!("Config structure test completed successfully - unified CollectionConfig approach working"); + assert_eq!(config.transactions.probability, 0.0); + + // Test that we can create a state update with incremental values + let slot = 1000; + let base_seq = 500; + let base_leaf_index = 100; + let simple_state_update = + get_rnd_state_update(&mut rng, &config, slot, base_seq, base_leaf_index); + println!("simple_state_update {:?}", simple_state_update); + + // Persist the simple state update + let result = persist_state_update_and_commit(&setup.db_conn, simple_state_update.clone()).await; + + // Should complete successfully + assert!( + result.is_ok(), + "Failed to persist simple state update: {:?}", + result.err() + ); + + // Assert that all output accounts were persisted correctly + assert_output_accounts_persisted(&setup.db_conn, &simple_state_update) + .await + .expect("Failed to verify output accounts persistence"); + + println!("Config structure test completed successfully - unified CollectionConfig approach with incremental slot/seq/leaf_index working"); } From 0916aa42ee30e1a90884f1dc21d0045397afbf07 Mon Sep 17 00:00:00 2001 From: ananas-block Date: Wed, 25 Jun 2025 00:42:42 +0100 Subject: [PATCH 04/21] add state root assert --- Cargo.toml | 1 + .../persist_state_update_test.rs | 86 +++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index b0e81c70..e46e5ea0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -132,6 +132,7 @@ rust-s3 = "0.34.0" function_name = "0.3.0" serial_test = "2.0.0" light-merkle-tree-reference = "2.0.0" +light-hasher = "3.1.0" [profile.dev] # Do not produce debug info for ~40% faster incremental compilation. diff --git a/tests/integration_tests/persist_state_update_test.rs b/tests/integration_tests/persist_state_update_test.rs index 82679ac0..c2f413c3 100644 --- a/tests/integration_tests/persist_state_update_test.rs +++ b/tests/integration_tests/persist_state_update_test.rs @@ -1,6 +1,8 @@ use crate::utils::*; use function_name::named; use light_compressed_account::indexer_event::event::BatchNullifyContext; +use light_merkle_tree_reference::MerkleTree; +use light_hasher::Poseidon; use photon_indexer::common::typedefs::account::AccountData; use photon_indexer::common::typedefs::account::{Account, AccountContext, AccountWithContext}; use photon_indexer::common::typedefs::bs64_string::Base64String; @@ -371,6 +373,79 @@ async fn assert_output_accounts_persisted( Ok(()) } +/// Assert that state tree root matches reference implementation after appending new hashes +async fn assert_state_tree_root( + db_conn: &DatabaseConnection, + reference_tree: &mut MerkleTree, + state_update: &StateUpdate, +) -> Result<(), Box> { + use photon_indexer::dao::generated::state_trees; + use sea_orm::ColumnTrait; + + if state_update.out_accounts.is_empty() { + println!("✅ No output accounts - skipping state tree root verification"); + return Ok(()); + } + + // Get the tree pubkey from the first output account (all should use same tree) + let tree_pubkey_bytes = state_update.out_accounts[0].account.tree.0.to_bytes().to_vec(); + + // Append all new account hashes to reference tree + for account_with_context in &state_update.out_accounts { + let account_hash_bytes = account_with_context.account.hash.0.to_vec(); + let mut hash_array = [0u8; 32]; + hash_array.copy_from_slice(&account_hash_bytes); + reference_tree.append(&hash_array)?; + } + + // Get reference tree root + let reference_root = reference_tree.root(); + + // First, let's see what nodes are actually in the state_trees table + let all_nodes = state_trees::Entity::find() + .filter(state_trees::Column::Tree.eq(tree_pubkey_bytes.clone())) + .all(db_conn) + .await?; + + println!("All nodes in state_trees table for tree {:?}:", hex::encode(&tree_pubkey_bytes)); + for node in &all_nodes { + println!(" node_idx: {}, level: {}, leaf_idx: {:?}, seq: {:?}, hash: {:?}", + node.node_idx, node.level, node.leaf_idx, node.seq, hex::encode(&node.hash)); + } + + if all_nodes.is_empty() { + println!("✅ No state tree nodes found - this might be expected for the test configuration"); + return Ok(()); + } + + // Find the root node (highest level) + let max_level = all_nodes.iter().map(|node| node.level).max().unwrap_or(0); + let root_nodes: Vec<_> = all_nodes.iter().filter(|node| node.level == max_level).collect(); + + if root_nodes.len() != 1 { + println!("⚠️ Multiple or no root nodes found at level {}: {:?}", max_level, root_nodes.len()); + // For now, just skip the root verification + return Ok(()); + } + + let root_node = root_nodes[0]; + let mut db_root_array = [0u8; 32]; + db_root_array.copy_from_slice(&root_node.hash); + + assert_eq!( + reference_root, db_root_array, + "State tree root mismatch!\nReference: {:?}\nDatabase: {:?}", + reference_root, db_root_array + ); + + println!( + "✅ Successfully verified state tree root matches reference implementation" + ); + println!("Tree root: {:?}", reference_root); + + Ok(()) +} + #[named] #[rstest] #[tokio::test] @@ -447,6 +522,12 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat println!("\n\nconfig structure test seed {}\n\n", seed); let mut rng = StdRng::seed_from_u64(seed); + // Initialize reference Merkle tree for state tree root verification + let tree_info = TreeInfo::get(TEST_TREE_PUBKEY_STR) + .expect("Test tree should exist in QUEUE_TREE_MAPPING"); + let tree_height = tree_info.height as usize; + let mut reference_tree = MerkleTree::::new(tree_height, 0); + // Test that the new config structure works correctly let config = StateUpdateConfig::default(); @@ -486,5 +567,10 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat .await .expect("Failed to verify output accounts persistence"); + // Assert that state tree root matches reference implementation + assert_state_tree_root(&setup.db_conn, &mut reference_tree, &simple_state_update) + .await + .expect("Failed to verify state tree root"); + println!("Config structure test completed successfully - unified CollectionConfig approach with incremental slot/seq/leaf_index working"); } From b28feb9c542afcf419982b55289167dacbb6e7a3 Mon Sep 17 00:00:00 2001 From: ananas-block Date: Wed, 25 Jun 2025 00:44:49 +0100 Subject: [PATCH 05/21] add state root assert --- .../persist_state_update_test.rs | 128 ++++++++++++------ 1 file changed, 86 insertions(+), 42 deletions(-) diff --git a/tests/integration_tests/persist_state_update_test.rs b/tests/integration_tests/persist_state_update_test.rs index c2f413c3..88d34d1f 100644 --- a/tests/integration_tests/persist_state_update_test.rs +++ b/tests/integration_tests/persist_state_update_test.rs @@ -1,8 +1,9 @@ use crate::utils::*; use function_name::named; use light_compressed_account::indexer_event::event::BatchNullifyContext; -use light_merkle_tree_reference::MerkleTree; +use light_compressed_account::TreeType; use light_hasher::Poseidon; +use light_merkle_tree_reference::MerkleTree; use photon_indexer::common::typedefs::account::AccountData; use photon_indexer::common::typedefs::account::{Account, AccountContext, AccountWithContext}; use photon_indexer::common::typedefs::bs64_string::Base64String; @@ -75,7 +76,7 @@ impl Default for StateUpdateConfig { fn default() -> Self { Self { in_accounts: CollectionConfig::new(0, 5, 0.0), - out_accounts: CollectionConfig::new(0, 5, 1.0), + out_accounts: CollectionConfig::new(1, 5, 1.0), account_transactions: CollectionConfig::new(0, 3, 0.0), transactions: CollectionConfig::new(0, 2, 0.0), leaf_nullifications: CollectionConfig::new(0, 3, 0.0), @@ -151,7 +152,11 @@ fn get_rnd_state_update( seq: Some(UnsignedInteger(base_seq + i as u64)), slot_created: UnsignedInteger(slot), }, - context: AccountContext::default(), + context: AccountContext { + tree_type: TreeType::StateV1 as u16, + queue: tree_info.queue.into(), + ..Default::default() + }, }; state_update.out_accounts.push(account); } @@ -381,67 +386,106 @@ async fn assert_state_tree_root( ) -> Result<(), Box> { use photon_indexer::dao::generated::state_trees; use sea_orm::ColumnTrait; - if state_update.out_accounts.is_empty() { println!("✅ No output accounts - skipping state tree root verification"); return Ok(()); } // Get the tree pubkey from the first output account (all should use same tree) - let tree_pubkey_bytes = state_update.out_accounts[0].account.tree.0.to_bytes().to_vec(); - - // Append all new account hashes to reference tree + let tree_pubkey_bytes = state_update.out_accounts[0] + .account + .tree + .0 + .to_bytes() + .to_vec(); + + println!("Output Account Hashes:"); for account_with_context in &state_update.out_accounts { - let account_hash_bytes = account_with_context.account.hash.0.to_vec(); - let mut hash_array = [0u8; 32]; - hash_array.copy_from_slice(&account_hash_bytes); - reference_tree.append(&hash_array)?; + let account_hash = hex::encode(&account_with_context.account.hash.0); + let leaf_index = account_with_context.account.leaf_index.0; + println!(" Hash({}) at leaf_index {}", account_hash, leaf_index); } - // Get reference tree root - let reference_root = reference_tree.root(); - - // First, let's see what nodes are actually in the state_trees table - let all_nodes = state_trees::Entity::find() + // First, get all leaf nodes from database to verify they match our output accounts + let leaf_nodes = state_trees::Entity::find() .filter(state_trees::Column::Tree.eq(tree_pubkey_bytes.clone())) + .filter(state_trees::Column::Level.eq(0i64)) // Leaf level .all(db_conn) .await?; - println!("All nodes in state_trees table for tree {:?}:", hex::encode(&tree_pubkey_bytes)); - for node in &all_nodes { - println!(" node_idx: {}, level: {}, leaf_idx: {:?}, seq: {:?}, hash: {:?}", - node.node_idx, node.level, node.leaf_idx, node.seq, hex::encode(&node.hash)); + println!("Database Leaf Hashes:"); + for leaf in &leaf_nodes { + println!(" Hash({}) at leaf_idx={:?}", hex::encode(&leaf.hash), leaf.leaf_idx); } - if all_nodes.is_empty() { - println!("✅ No state tree nodes found - this might be expected for the test configuration"); - return Ok(()); + // Assert that all our new account hashes are present as leaf nodes in the database + for account_with_context in &state_update.out_accounts { + let account_hash = hex::encode(&account_with_context.account.hash.0); + let leaf_index = account_with_context.account.leaf_index.0; + + let found_leaf = leaf_nodes.iter().find(|leaf| { + leaf.leaf_idx == Some(leaf_index as i64) && + hex::encode(&leaf.hash) == account_hash + }); + + assert!(found_leaf.is_some(), + "Account hash {} at leaf_index {} not found in database leaf nodes", + account_hash, leaf_index); } + println!("✅ All account hashes verified as leaf nodes in database"); - // Find the root node (highest level) - let max_level = all_nodes.iter().map(|node| node.level).max().unwrap_or(0); - let root_nodes: Vec<_> = all_nodes.iter().filter(|node| node.level == max_level).collect(); - - if root_nodes.len() != 1 { - println!("⚠️ Multiple or no root nodes found at level {}: {:?}", max_level, root_nodes.len()); - // For now, just skip the root verification - return Ok(()); + // Construct reference tree from output accounts directly + // Find the maximum leaf index to determine tree size needed + let max_leaf_idx = state_update.out_accounts + .iter() + .map(|acc| acc.account.leaf_index.0) + .max() + .unwrap_or(0); + + println!("Constructing reference tree up to leaf index {}", max_leaf_idx); + + // Append leaves to reference tree in the correct positions + // Fill with zero hashes for missing leaves, actual account hashes for present ones + for i in 0..=max_leaf_idx { + let leaf_hash = state_update.out_accounts + .iter() + .find(|acc| acc.account.leaf_index.0 == i) + .map(|acc| acc.account.hash.0) + .unwrap_or([0u8; 32]); // Zero hash for missing leaves + + reference_tree.append(&leaf_hash)?; } + // Get reference tree root after construction + let reference_root = reference_tree.root(); + println!("Reference tree root: {}", hex::encode(&reference_root)); + + // Get database root node for comparison + let all_nodes = state_trees::Entity::find() + .filter(state_trees::Column::Tree.eq(tree_pubkey_bytes.clone())) + .all(db_conn) + .await?; + + let max_level = all_nodes.iter().map(|node| node.level).max().unwrap_or(0); + let root_nodes: Vec<_> = all_nodes + .iter() + .filter(|node| node.level == max_level) + .collect(); + + assert_eq!(root_nodes.len(), 1, "Expected exactly 1 root node, found {}", root_nodes.len()); + let root_node = root_nodes[0]; let mut db_root_array = [0u8; 32]; db_root_array.copy_from_slice(&root_node.hash); + println!("Database root: {}", hex::encode(&db_root_array)); assert_eq!( reference_root, db_root_array, - "State tree root mismatch!\nReference: {:?}\nDatabase: {:?}", - reference_root, db_root_array + "State tree root mismatch!\nReference: {}\nDatabase: {}", + hex::encode(&reference_root), hex::encode(&db_root_array) ); - - println!( - "✅ Successfully verified state tree root matches reference implementation" - ); - println!("Tree root: {:?}", reference_root); + + println!("✅ State tree root verification successful!"); Ok(()) } @@ -523,10 +567,10 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat let mut rng = StdRng::seed_from_u64(seed); // Initialize reference Merkle tree for state tree root verification - let tree_info = TreeInfo::get(TEST_TREE_PUBKEY_STR) - .expect("Test tree should exist in QUEUE_TREE_MAPPING"); + let tree_info = + TreeInfo::get(TEST_TREE_PUBKEY_STR).expect("Test tree should exist in QUEUE_TREE_MAPPING"); let tree_height = tree_info.height as usize; - let mut reference_tree = MerkleTree::::new(tree_height, 0); + let mut reference_tree = MerkleTree::::new(26, 0); // Test that the new config structure works correctly let config = StateUpdateConfig::default(); @@ -536,7 +580,7 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat assert_eq!(config.in_accounts.max_entries, 5); assert_eq!(config.in_accounts.probability, 0.0); - assert_eq!(config.out_accounts.min_entries, 0); + assert_eq!(config.out_accounts.min_entries, 1); assert_eq!(config.out_accounts.max_entries, 5); assert_eq!(config.out_accounts.probability, 1.0); From cb2e2000fbebd8fc1401adbad698982e3e41390d Mon Sep 17 00:00:00 2001 From: ananas-block Date: Wed, 25 Jun 2025 01:11:36 +0100 Subject: [PATCH 06/21] add iters --- .../persist_state_update_test.rs | 145 +++++++++--------- 1 file changed, 74 insertions(+), 71 deletions(-) diff --git a/tests/integration_tests/persist_state_update_test.rs b/tests/integration_tests/persist_state_update_test.rs index 88d34d1f..16581fad 100644 --- a/tests/integration_tests/persist_state_update_test.rs +++ b/tests/integration_tests/persist_state_update_test.rs @@ -76,7 +76,7 @@ impl Default for StateUpdateConfig { fn default() -> Self { Self { in_accounts: CollectionConfig::new(0, 5, 0.0), - out_accounts: CollectionConfig::new(1, 5, 1.0), + out_accounts: CollectionConfig::new(0, 5, 1.0), account_transactions: CollectionConfig::new(0, 3, 0.0), transactions: CollectionConfig::new(0, 2, 0.0), leaf_nullifications: CollectionConfig::new(0, 3, 0.0), @@ -299,12 +299,12 @@ async fn assert_output_accounts_persisted( if state_update.out_accounts.is_empty() { // If no accounts expected, verify table is empty - let account_count = accounts::Entity::find().count(db_conn).await?; - assert_eq!( - account_count, 0, - "Expected no accounts in database, but found {}", - account_count - ); + // let account_count = accounts::Entity::find().count(db_conn).await?; + // assert_eq!( + // account_count, 0, + // "Expected no accounts in database, but found {}", + // account_count + // ); return Ok(()); } @@ -409,50 +409,45 @@ async fn assert_state_tree_root( // First, get all leaf nodes from database to verify they match our output accounts let leaf_nodes = state_trees::Entity::find() .filter(state_trees::Column::Tree.eq(tree_pubkey_bytes.clone())) - .filter(state_trees::Column::Level.eq(0i64)) // Leaf level + .filter(state_trees::Column::Level.eq(0i64)) // Leaf level .all(db_conn) .await?; println!("Database Leaf Hashes:"); for leaf in &leaf_nodes { - println!(" Hash({}) at leaf_idx={:?}", hex::encode(&leaf.hash), leaf.leaf_idx); + println!( + " Hash({}) at leaf_idx={:?}", + hex::encode(&leaf.hash), + leaf.leaf_idx + ); } // Assert that all our new account hashes are present as leaf nodes in the database for account_with_context in &state_update.out_accounts { let account_hash = hex::encode(&account_with_context.account.hash.0); let leaf_index = account_with_context.account.leaf_index.0; - + let found_leaf = leaf_nodes.iter().find(|leaf| { - leaf.leaf_idx == Some(leaf_index as i64) && - hex::encode(&leaf.hash) == account_hash + leaf.leaf_idx == Some(leaf_index as i64) && hex::encode(&leaf.hash) == account_hash }); - - assert!(found_leaf.is_some(), - "Account hash {} at leaf_index {} not found in database leaf nodes", - account_hash, leaf_index); + + assert!( + found_leaf.is_some(), + "Account hash {} at leaf_index {} not found in database leaf nodes", + account_hash, + leaf_index + ); } println!("✅ All account hashes verified as leaf nodes in database"); - // Construct reference tree from output accounts directly - // Find the maximum leaf index to determine tree size needed - let max_leaf_idx = state_update.out_accounts - .iter() - .map(|acc| acc.account.leaf_index.0) - .max() - .unwrap_or(0); - - println!("Constructing reference tree up to leaf index {}", max_leaf_idx); - - // Append leaves to reference tree in the correct positions - // Fill with zero hashes for missing leaves, actual account hashes for present ones - for i in 0..=max_leaf_idx { - let leaf_hash = state_update.out_accounts - .iter() - .find(|acc| acc.account.leaf_index.0 == i) - .map(|acc| acc.account.hash.0) - .unwrap_or([0u8; 32]); // Zero hash for missing leaves - + // Append only the new leaves from current state update to reference tree + println!( + "Appending {} new leaves from current state update", + state_update.out_accounts.len() + ); + + for account_with_context in &state_update.out_accounts { + let leaf_hash = account_with_context.account.hash.0; reference_tree.append(&leaf_hash)?; } @@ -472,19 +467,26 @@ async fn assert_state_tree_root( .filter(|node| node.level == max_level) .collect(); - assert_eq!(root_nodes.len(), 1, "Expected exactly 1 root node, found {}", root_nodes.len()); - + assert_eq!( + root_nodes.len(), + 1, + "Expected exactly 1 root node, found {}", + root_nodes.len() + ); + let root_node = root_nodes[0]; let mut db_root_array = [0u8; 32]; db_root_array.copy_from_slice(&root_node.hash); println!("Database root: {}", hex::encode(&db_root_array)); assert_eq!( - reference_root, db_root_array, + reference_root, + db_root_array, "State tree root mismatch!\nReference: {}\nDatabase: {}", - hex::encode(&reference_root), hex::encode(&db_root_array) + hex::encode(&reference_root), + hex::encode(&db_root_array) ); - + println!("✅ State tree root verification successful!"); Ok(()) @@ -566,10 +568,6 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat println!("\n\nconfig structure test seed {}\n\n", seed); let mut rng = StdRng::seed_from_u64(seed); - // Initialize reference Merkle tree for state tree root verification - let tree_info = - TreeInfo::get(TEST_TREE_PUBKEY_STR).expect("Test tree should exist in QUEUE_TREE_MAPPING"); - let tree_height = tree_info.height as usize; let mut reference_tree = MerkleTree::::new(26, 0); // Test that the new config structure works correctly @@ -580,7 +578,7 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat assert_eq!(config.in_accounts.max_entries, 5); assert_eq!(config.in_accounts.probability, 0.0); - assert_eq!(config.out_accounts.min_entries, 1); + assert_eq!(config.out_accounts.min_entries, 0); assert_eq!(config.out_accounts.max_entries, 5); assert_eq!(config.out_accounts.probability, 1.0); @@ -589,32 +587,37 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat assert_eq!(config.transactions.probability, 0.0); // Test that we can create a state update with incremental values - let slot = 1000; - let base_seq = 500; - let base_leaf_index = 100; - let simple_state_update = - get_rnd_state_update(&mut rng, &config, slot, base_seq, base_leaf_index); - println!("simple_state_update {:?}", simple_state_update); - - // Persist the simple state update - let result = persist_state_update_and_commit(&setup.db_conn, simple_state_update.clone()).await; - - // Should complete successfully - assert!( - result.is_ok(), - "Failed to persist simple state update: {:?}", - result.err() - ); - - // Assert that all output accounts were persisted correctly - assert_output_accounts_persisted(&setup.db_conn, &simple_state_update) - .await - .expect("Failed to verify output accounts persistence"); - - // Assert that state tree root matches reference implementation - assert_state_tree_root(&setup.db_conn, &mut reference_tree, &simple_state_update) - .await - .expect("Failed to verify state tree root"); + let mut base_seq = 500; + let mut base_leaf_index = 0; + let num_iters = 1000; + for slot in 0..num_iters { + println!("iter {}", slot); + let simple_state_update = + get_rnd_state_update(&mut rng, &config, slot, base_seq, base_leaf_index); + println!("simple_state_update {:?}", simple_state_update); + + // Persist the simple state update + let result = + persist_state_update_and_commit(&setup.db_conn, simple_state_update.clone()).await; + + // Should complete successfully + assert!( + result.is_ok(), + "Failed to persist simple state update: {:?}", + result.err() + ); + // Assert that all output accounts were persisted correctly + assert_output_accounts_persisted(&setup.db_conn, &simple_state_update) + .await + .expect("Failed to verify output accounts persistence"); + + // Assert that state tree root matches reference implementation + assert_state_tree_root(&setup.db_conn, &mut reference_tree, &simple_state_update) + .await + .expect("Failed to verify state tree root"); + base_seq += simple_state_update.out_accounts.len() as u64; + base_leaf_index += simple_state_update.out_accounts.len() as u64; + } println!("Config structure test completed successfully - unified CollectionConfig approach with incremental slot/seq/leaf_index working"); } From fa813d3c33ef80fa7b7fe7ec5cf9f8ca8b43bea9 Mon Sep 17 00:00:00 2001 From: ananas-block Date: Wed, 25 Jun 2025 14:39:21 +0100 Subject: [PATCH 07/21] stash --- .../persist_state_update_test.rs | 98 ++++++++++++++++--- 1 file changed, 82 insertions(+), 16 deletions(-) diff --git a/tests/integration_tests/persist_state_update_test.rs b/tests/integration_tests/persist_state_update_test.rs index 16581fad..88977027 100644 --- a/tests/integration_tests/persist_state_update_test.rs +++ b/tests/integration_tests/persist_state_update_test.rs @@ -30,7 +30,9 @@ use solana_sdk::signature::Signature; use std::env; // Use the specific tree from QUEUE_TREE_MAPPING -const TEST_TREE_PUBKEY_STR: &str = "smt1NamzXdq4AMqS2fS2F1i5KTYPZRhoHgWx38d8WsT"; +const V1_TEST_TREE_PUBKEY_STR: &str = "smt1NamzXdq4AMqS2fS2F1i5KTYPZRhoHgWx38d8WsT"; +const V2_TEST_TREE_PUBKEY_STR: &str = "HLKs5NJ8FXkJg8BrzJt56adFYYuwg5etzDtBbQYTsixu"; +const V2_TEST_QUEUE_PUBKEY_STR: &str = "HLKs5NJ8FXkJg8BrzJt56adFYYuwg5etzDtBbQYTsixu"; /// Configuration for generating random collections #[derive(Debug, Clone)] @@ -55,7 +57,8 @@ impl CollectionConfig { pub struct StateUpdateConfig { // Collection configurations for StateUpdate fields pub in_accounts: CollectionConfig, - pub out_accounts: CollectionConfig, + pub out_accounts_v1: CollectionConfig, + pub out_accounts_v2: CollectionConfig, pub account_transactions: CollectionConfig, pub transactions: CollectionConfig, pub leaf_nullifications: CollectionConfig, @@ -76,7 +79,8 @@ impl Default for StateUpdateConfig { fn default() -> Self { Self { in_accounts: CollectionConfig::new(0, 5, 0.0), - out_accounts: CollectionConfig::new(0, 5, 1.0), + out_accounts_v1: CollectionConfig::new(0, 5, 1.0), + out_accounts_v2: CollectionConfig::new(0, 5, 1.0), account_transactions: CollectionConfig::new(0, 3, 0.0), transactions: CollectionConfig::new(0, 2, 0.0), leaf_nullifications: CollectionConfig::new(0, 3, 0.0), @@ -104,11 +108,6 @@ fn get_rnd_state_update( ) -> StateUpdate { let mut state_update = StateUpdate::default(); - // Get tree info from QUEUE_TREE_MAPPING - let tree_info = - TreeInfo::get(TEST_TREE_PUBKEY_STR).expect("Test tree should exist in QUEUE_TREE_MAPPING"); - let test_tree_pubkey = tree_info.tree; - // Generate in_accounts (HashSet) if rng.gen_bool(config.in_accounts.probability) { let count = rng.gen_range(config.in_accounts.min_entries..=config.in_accounts.max_entries); @@ -118,9 +117,14 @@ fn get_rnd_state_update( } // Generate out_accounts (Vec) - if rng.gen_bool(config.out_accounts.probability) { + if rng.gen_bool(config.out_accounts_v1.probability) { + // Get tree info from QUEUE_TREE_MAPPING + let tree_info = TreeInfo::get(V1_TEST_TREE_PUBKEY_STR) + .expect("Test tree should exist in QUEUE_TREE_MAPPING"); + let test_tree_pubkey = tree_info.tree; + let count = - rng.gen_range(config.out_accounts.min_entries..=config.out_accounts.max_entries); + rng.gen_range(config.out_accounts_v1.min_entries..=config.out_accounts_v1.max_entries); for i in 0..count { let account = AccountWithContext { account: Account { @@ -162,6 +166,68 @@ fn get_rnd_state_update( } } + // Generate out_accounts (Vec) + if rng.gen_bool(config.out_accounts_v2.probability) { + // Get tree info from QUEUE_TREE_MAPPING + let tree_info = TreeInfo::get(V2_TEST_TREE_PUBKEY_STR) + .expect("Test tree should exist in QUEUE_TREE_MAPPING"); + let test_tree_pubkey = tree_info.tree; + + let count = + rng.gen_range(config.out_accounts_v2.min_entries..=config.out_accounts_v2.max_entries); + for i in 0..count { + let account = AccountWithContext { + account: Account { + hash: Hash::new_unique(), + address: if rng.gen_bool(0.7) { + Some(SerializablePubkey::new_unique()) + } else { + None + }, + data: if rng.gen_bool(0.6) { + let data_size = rng.gen_range(config.data_size_min..=config.data_size_max); + Some(AccountData { + discriminator: UnsignedInteger( + rng.gen_range(config.discriminator_min..=config.discriminator_max), + ), + data: Base64String((0..data_size).map(|_| rng.gen()).collect()), + data_hash: Hash::new_unique(), + }) + } else { + None + }, + owner: SerializablePubkey::new_unique(), + lamports: UnsignedInteger( + rng.gen_range(config.lamports_min as i64..=config.lamports_max as i64) + as u64, + ), + tree: SerializablePubkey::from(test_tree_pubkey), + leaf_index: UnsignedInteger(base_leaf_index + i as u64), + seq: Some(UnsignedInteger(base_seq + i as u64)), + slot_created: UnsignedInteger(slot), + }, + context: AccountContext { + tree_type: TreeType::StateV2 as u16, + queue: tree_info.queue.into(), + in_output_queue: true, + tx_hash: if rng.gen_bool(0.5) { + Some(rng.gen::<[u8; 32]>().into()) + } else { + None + }, + ..Default::default() + }, + }; + state_update.out_accounts.push(account); + } + } + + // Kept until we introduce v1 and v2 differentiation for nullification + // Get tree info from QUEUE_TREE_MAPPING + let tree_info = TreeInfo::get(V1_TEST_TREE_PUBKEY_STR) + .expect("Test tree should exist in QUEUE_TREE_MAPPING"); + let test_tree_pubkey = tree_info.tree; + // Generate account_transactions (HashSet) if rng.gen_bool(config.account_transactions.probability) { let count = rng.gen_range( @@ -341,7 +407,7 @@ async fn assert_output_accounts_persisted( in_output_queue: context.in_output_queue, // From account context queue: context.queue.0.to_bytes().to_vec(), // Use queue from account context nullifier: None, // Default value - tx_hash: None, // Default value + tx_hash: context.tx_hash.as_ref().map(|hash| hash.0.to_vec()), } }) .collect(); @@ -401,7 +467,7 @@ async fn assert_state_tree_root( println!("Output Account Hashes:"); for account_with_context in &state_update.out_accounts { - let account_hash = hex::encode(&account_with_context.account.hash.0); + let account_hash = hex::encode(account_with_context.account.hash.0); let leaf_index = account_with_context.account.leaf_index.0; println!(" Hash({}) at leaf_index {}", account_hash, leaf_index); } @@ -578,9 +644,9 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat assert_eq!(config.in_accounts.max_entries, 5); assert_eq!(config.in_accounts.probability, 0.0); - assert_eq!(config.out_accounts.min_entries, 0); - assert_eq!(config.out_accounts.max_entries, 5); - assert_eq!(config.out_accounts.probability, 1.0); + assert_eq!(config.out_accounts_v1.min_entries, 0); + assert_eq!(config.out_accounts_v1.max_entries, 5); + assert_eq!(config.out_accounts_v1.probability, 1.0); assert_eq!(config.transactions.min_entries, 0); assert_eq!(config.transactions.max_entries, 2); @@ -589,7 +655,7 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat // Test that we can create a state update with incremental values let mut base_seq = 500; let mut base_leaf_index = 0; - let num_iters = 1000; + let num_iters = 10; for slot in 0..num_iters { println!("iter {}", slot); let simple_state_update = From 4e2e7c7b6ca1bf6d83fa3560e161a355f75d49b4 Mon Sep 17 00:00:00 2001 From: ananas-block Date: Wed, 25 Jun 2025 14:50:32 +0100 Subject: [PATCH 08/21] stash failing accounts test --- .../persist_state_update_test.rs | 123 +++++++++++++----- 1 file changed, 94 insertions(+), 29 deletions(-) diff --git a/tests/integration_tests/persist_state_update_test.rs b/tests/integration_tests/persist_state_update_test.rs index 88977027..b844478d 100644 --- a/tests/integration_tests/persist_state_update_test.rs +++ b/tests/integration_tests/persist_state_update_test.rs @@ -29,10 +29,10 @@ use serial_test::serial; use solana_sdk::signature::Signature; use std::env; -// Use the specific tree from QUEUE_TREE_MAPPING +// Use the specific trees from QUEUE_TREE_MAPPING const V1_TEST_TREE_PUBKEY_STR: &str = "smt1NamzXdq4AMqS2fS2F1i5KTYPZRhoHgWx38d8WsT"; const V2_TEST_TREE_PUBKEY_STR: &str = "HLKs5NJ8FXkJg8BrzJt56adFYYuwg5etzDtBbQYTsixu"; -const V2_TEST_QUEUE_PUBKEY_STR: &str = "HLKs5NJ8FXkJg8BrzJt56adFYYuwg5etzDtBbQYTsixu"; +const V2_TEST_QUEUE_PUBKEY_STR: &str = "6L7SzhYB3anwEQ9cphpJ1U7Scwj57bx2xueReg7R9cKU"; /// Configuration for generating random collections #[derive(Debug, Clone)] @@ -104,7 +104,8 @@ fn get_rnd_state_update( config: &StateUpdateConfig, slot: u64, base_seq: u64, - base_leaf_index: u64, + base_leaf_index_v1: u64, + base_leaf_index_v2: u64, ) -> StateUpdate { let mut state_update = StateUpdate::default(); @@ -152,13 +153,15 @@ fn get_rnd_state_update( as u64, ), tree: SerializablePubkey::from(test_tree_pubkey), - leaf_index: UnsignedInteger(base_leaf_index + i as u64), + leaf_index: UnsignedInteger(base_leaf_index_v1 + i as u64), seq: Some(UnsignedInteger(base_seq + i as u64)), slot_created: UnsignedInteger(slot), }, context: AccountContext { tree_type: TreeType::StateV1 as u16, queue: tree_info.queue.into(), + tx_hash: None, // V1 accounts never have tx_hash + in_output_queue: false, // V1 accounts don't use output queues ..Default::default() }, }; @@ -202,16 +205,16 @@ fn get_rnd_state_update( as u64, ), tree: SerializablePubkey::from(test_tree_pubkey), - leaf_index: UnsignedInteger(base_leaf_index + i as u64), - seq: Some(UnsignedInteger(base_seq + i as u64)), + leaf_index: UnsignedInteger(base_leaf_index_v2 + i as u64), + seq: None, // V2 accounts in output queue don't have seq initially slot_created: UnsignedInteger(slot), }, context: AccountContext { tree_type: TreeType::StateV2 as u16, queue: tree_info.queue.into(), - in_output_queue: true, + in_output_queue: true, // V2 accounts use output queues tx_hash: if rng.gen_bool(0.5) { - Some(rng.gen::<[u8; 32]>().into()) + Some(Hash::new_unique()) // Generate proper Hash for V2 } else { None }, @@ -275,7 +278,7 @@ fn get_rnd_state_update( rng.fill(&mut sig_bytes); state_update.leaf_nullifications.insert(LeafNullification { tree: test_tree_pubkey, - leaf_index: base_leaf_index + i as u64, + leaf_index: base_leaf_index_v1 + i as u64, seq: base_seq + i as u64, signature: Signature::from(sig_bytes), }); @@ -290,14 +293,14 @@ fn get_rnd_state_update( ); for i in 0..count { let tree = test_tree_pubkey; - let index = base_leaf_index + i as u64; + let index = base_leaf_index_v1 + i as u64; let update = IndexedTreeLeafUpdate { tree, leaf: RawIndexedElement { value: rng.gen::<[u8; 32]>(), next_index: rng.gen::() as usize, next_value: rng.gen::<[u8; 32]>(), - index: (base_leaf_index + i as u64) as usize, + index: (base_leaf_index_v1 + i as u64) as usize, }, hash: rng.gen::<[u8; 32]>(), seq: base_seq + i as u64, @@ -457,29 +460,54 @@ async fn assert_state_tree_root( return Ok(()); } - // Get the tree pubkey from the first output account (all should use same tree) - let tree_pubkey_bytes = state_update.out_accounts[0] + // For now, only verify v1 accounts since we're using a single reference tree + // Filter to only v1 accounts for tree root verification + let v1_accounts: Vec<_> = state_update.out_accounts.iter() + .filter(|acc| acc.context.tree_type == TreeType::StateV1 as u16) + .collect(); + + if v1_accounts.is_empty() { + println!("✅ No v1 output accounts - skipping state tree root verification"); + return Ok(()); + } + + // Get the tree pubkey from the first v1 output account + let tree_pubkey_bytes = v1_accounts[0] .account .tree .0 .to_bytes() .to_vec(); - println!("Output Account Hashes:"); - for account_with_context in &state_update.out_accounts { + println!("V1 Account Hashes (should be in tree):"); + for account_with_context in &v1_accounts { let account_hash = hex::encode(account_with_context.account.hash.0); let leaf_index = account_with_context.account.leaf_index.0; - println!(" Hash({}) at leaf_index {}", account_hash, leaf_index); + println!(" V1 Hash({}) at leaf_index {}", account_hash, leaf_index); + } + + // Also log V2 accounts for visibility (these go to output queue) + let v2_accounts: Vec<_> = state_update.out_accounts.iter() + .filter(|acc| acc.context.tree_type == TreeType::StateV2 as u16) + .collect(); + + if !v2_accounts.is_empty() { + println!("V2 Account Hashes (go to output queue, not tree directly):"); + for account_with_context in &v2_accounts { + let account_hash = hex::encode(account_with_context.account.hash.0); + let leaf_index = account_with_context.account.leaf_index.0; + println!(" V2 Hash({}) at leaf_index {}", account_hash, leaf_index); + } } - // First, get all leaf nodes from database to verify they match our output accounts + // First, get all leaf nodes from database to verify they match our V1 output accounts let leaf_nodes = state_trees::Entity::find() .filter(state_trees::Column::Tree.eq(tree_pubkey_bytes.clone())) .filter(state_trees::Column::Level.eq(0i64)) // Leaf level .all(db_conn) .await?; - println!("Database Leaf Hashes:"); + println!("Database Leaf Hashes (for V1 tree):"); for leaf in &leaf_nodes { println!( " Hash({}) at leaf_idx={:?}", @@ -488,8 +516,8 @@ async fn assert_state_tree_root( ); } - // Assert that all our new account hashes are present as leaf nodes in the database - for account_with_context in &state_update.out_accounts { + // Assert that all our V1 account hashes are present as leaf nodes in the database + for account_with_context in &v1_accounts { let account_hash = hex::encode(&account_with_context.account.hash.0); let leaf_index = account_with_context.account.leaf_index.0; @@ -499,20 +527,43 @@ async fn assert_state_tree_root( assert!( found_leaf.is_some(), - "Account hash {} at leaf_index {} not found in database leaf nodes", + "V1 account hash {} at leaf_index {} not found in database leaf nodes", account_hash, leaf_index ); } - println!("✅ All account hashes verified as leaf nodes in database"); + println!("✅ All V1 account hashes verified as leaf nodes in database"); + + // Verify V2 accounts are NOT in the state tree (they should be in output queue only) + if !v2_accounts.is_empty() { + println!("Verifying V2 accounts are NOT in state tree (should be in output queue only):"); + for account_with_context in &v2_accounts { + let account_hash = hex::encode(&account_with_context.account.hash.0); + let leaf_index = account_with_context.account.leaf_index.0; + + let found_leaf = leaf_nodes.iter().find(|leaf| { + hex::encode(&leaf.hash) == account_hash + }); + + assert!( + found_leaf.is_none(), + "V2 account hash {} should NOT be found in state tree leaf nodes (should be in output queue only), but was found at leaf_idx={:?}", + account_hash, + found_leaf.map(|leaf| leaf.leaf_idx) + ); + + println!(" ✅ V2 Hash({}) correctly NOT in state tree (in output queue)", account_hash); + } + println!("✅ All V2 account hashes verified as NOT in state tree (correctly in output queue)"); + } - // Append only the new leaves from current state update to reference tree + // Append only the V1 leaves from current state update to reference tree println!( - "Appending {} new leaves from current state update", - state_update.out_accounts.len() + "Appending {} V1 leaves from current state update to reference tree", + v1_accounts.len() ); - for account_with_context in &state_update.out_accounts { + for account_with_context in &v1_accounts { let leaf_hash = account_with_context.account.hash.0; reference_tree.append(&leaf_hash)?; } @@ -648,18 +699,23 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat assert_eq!(config.out_accounts_v1.max_entries, 5); assert_eq!(config.out_accounts_v1.probability, 1.0); + assert_eq!(config.out_accounts_v2.min_entries, 0); + assert_eq!(config.out_accounts_v2.max_entries, 5); + assert_eq!(config.out_accounts_v2.probability, 1.0); + assert_eq!(config.transactions.min_entries, 0); assert_eq!(config.transactions.max_entries, 2); assert_eq!(config.transactions.probability, 0.0); // Test that we can create a state update with incremental values let mut base_seq = 500; - let mut base_leaf_index = 0; + let mut base_leaf_index_v1 = 0; + let mut base_leaf_index_v2 = 1000; // Use separate leaf index space for v2 let num_iters = 10; for slot in 0..num_iters { println!("iter {}", slot); let simple_state_update = - get_rnd_state_update(&mut rng, &config, slot, base_seq, base_leaf_index); + get_rnd_state_update(&mut rng, &config, slot, base_seq, base_leaf_index_v1, base_leaf_index_v2); println!("simple_state_update {:?}", simple_state_update); // Persist the simple state update @@ -682,8 +738,17 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat assert_state_tree_root(&setup.db_conn, &mut reference_tree, &simple_state_update) .await .expect("Failed to verify state tree root"); + // Update leaf indices separately for v1 and v2 + let v1_count = simple_state_update.out_accounts.iter() + .filter(|acc| acc.context.tree_type == TreeType::StateV1 as u16) + .count() as u64; + let v2_count = simple_state_update.out_accounts.iter() + .filter(|acc| acc.context.tree_type == TreeType::StateV2 as u16) + .count() as u64; + base_seq += simple_state_update.out_accounts.len() as u64; - base_leaf_index += simple_state_update.out_accounts.len() as u64; + base_leaf_index_v1 += v1_count; + base_leaf_index_v2 += v2_count; } println!("Config structure test completed successfully - unified CollectionConfig approach with incremental slot/seq/leaf_index working"); } From 8ca03ca975a4b7aa27cdc3b921b69623165ccad9 Mon Sep 17 00:00:00 2001 From: ananas-block Date: Wed, 25 Jun 2025 15:06:49 +0100 Subject: [PATCH 09/21] fix: append_output_accounts set tx hash --- src/ingester/persist/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ingester/persist/mod.rs b/src/ingester/persist/mod.rs index 16aee808..01cdee4a 100644 --- a/src/ingester/persist/mod.rs +++ b/src/ingester/persist/mod.rs @@ -435,7 +435,7 @@ async fn append_output_accounts( slot_created: Set(account.account.slot_created.0 as i64), seq: Set(account.account.seq.map(|x| x.0 as i64)), prev_spent: Set(None), - tx_hash: Default::default(), // Its sets at input queue insertion for batch updates + tx_hash: Set(account.context.tx_hash.as_ref().map(|x| x.to_vec())), }); if let Some(token_data) = parse_token_data(&account.account)? { From 8fa9ac3fd528067f6527e677980daf16e795d567 Mon Sep 17 00:00:00 2001 From: ananas-block Date: Wed, 25 Jun 2025 15:07:59 +0100 Subject: [PATCH 10/21] add v2 output accounts to test --- .../persist_state_update_test.rs | 61 +++++++++++-------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/tests/integration_tests/persist_state_update_test.rs b/tests/integration_tests/persist_state_update_test.rs index b844478d..2e64444a 100644 --- a/tests/integration_tests/persist_state_update_test.rs +++ b/tests/integration_tests/persist_state_update_test.rs @@ -160,7 +160,7 @@ fn get_rnd_state_update( context: AccountContext { tree_type: TreeType::StateV1 as u16, queue: tree_info.queue.into(), - tx_hash: None, // V1 accounts never have tx_hash + tx_hash: None, // V1 accounts never have tx_hash in_output_queue: false, // V1 accounts don't use output queues ..Default::default() }, @@ -214,7 +214,7 @@ fn get_rnd_state_update( queue: tree_info.queue.into(), in_output_queue: true, // V2 accounts use output queues tx_hash: if rng.gen_bool(0.5) { - Some(Hash::new_unique()) // Generate proper Hash for V2 + Some(Hash::from(rng.gen::<[u8; 32]>())) } else { None }, @@ -462,22 +462,19 @@ async fn assert_state_tree_root( // For now, only verify v1 accounts since we're using a single reference tree // Filter to only v1 accounts for tree root verification - let v1_accounts: Vec<_> = state_update.out_accounts.iter() + let v1_accounts: Vec<_> = state_update + .out_accounts + .iter() .filter(|acc| acc.context.tree_type == TreeType::StateV1 as u16) .collect(); - + if v1_accounts.is_empty() { println!("✅ No v1 output accounts - skipping state tree root verification"); return Ok(()); } // Get the tree pubkey from the first v1 output account - let tree_pubkey_bytes = v1_accounts[0] - .account - .tree - .0 - .to_bytes() - .to_vec(); + let tree_pubkey_bytes = v1_accounts[0].account.tree.0.to_bytes().to_vec(); println!("V1 Account Hashes (should be in tree):"); for account_with_context in &v1_accounts { @@ -487,10 +484,12 @@ async fn assert_state_tree_root( } // Also log V2 accounts for visibility (these go to output queue) - let v2_accounts: Vec<_> = state_update.out_accounts.iter() + let v2_accounts: Vec<_> = state_update + .out_accounts + .iter() .filter(|acc| acc.context.tree_type == TreeType::StateV2 as u16) .collect(); - + if !v2_accounts.is_empty() { println!("V2 Account Hashes (go to output queue, not tree directly):"); for account_with_context in &v2_accounts { @@ -539,11 +538,10 @@ async fn assert_state_tree_root( println!("Verifying V2 accounts are NOT in state tree (should be in output queue only):"); for account_with_context in &v2_accounts { let account_hash = hex::encode(&account_with_context.account.hash.0); - let leaf_index = account_with_context.account.leaf_index.0; - let found_leaf = leaf_nodes.iter().find(|leaf| { - hex::encode(&leaf.hash) == account_hash - }); + let found_leaf = leaf_nodes + .iter() + .find(|leaf| hex::encode(&leaf.hash) == account_hash); assert!( found_leaf.is_none(), @@ -551,10 +549,15 @@ async fn assert_state_tree_root( account_hash, found_leaf.map(|leaf| leaf.leaf_idx) ); - - println!(" ✅ V2 Hash({}) correctly NOT in state tree (in output queue)", account_hash); + + println!( + " ✅ V2 Hash({}) correctly NOT in state tree (in output queue)", + account_hash + ); } - println!("✅ All V2 account hashes verified as NOT in state tree (correctly in output queue)"); + println!( + "✅ All V2 account hashes verified as NOT in state tree (correctly in output queue)" + ); } // Append only the V1 leaves from current state update to reference tree @@ -714,8 +717,14 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat let num_iters = 10; for slot in 0..num_iters { println!("iter {}", slot); - let simple_state_update = - get_rnd_state_update(&mut rng, &config, slot, base_seq, base_leaf_index_v1, base_leaf_index_v2); + let simple_state_update = get_rnd_state_update( + &mut rng, + &config, + slot, + base_seq, + base_leaf_index_v1, + base_leaf_index_v2, + ); println!("simple_state_update {:?}", simple_state_update); // Persist the simple state update @@ -739,13 +748,17 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat .await .expect("Failed to verify state tree root"); // Update leaf indices separately for v1 and v2 - let v1_count = simple_state_update.out_accounts.iter() + let v1_count = simple_state_update + .out_accounts + .iter() .filter(|acc| acc.context.tree_type == TreeType::StateV1 as u16) .count() as u64; - let v2_count = simple_state_update.out_accounts.iter() + let v2_count = simple_state_update + .out_accounts + .iter() .filter(|acc| acc.context.tree_type == TreeType::StateV2 as u16) .count() as u64; - + base_seq += simple_state_update.out_accounts.len() as u64; base_leaf_index_v1 += v1_count; base_leaf_index_v2 += v2_count; From 245412199f8dbda82b43aa50540d8d940baeb447 Mon Sep 17 00:00:00 2001 From: ananas-block Date: Wed, 25 Jun 2025 15:16:56 +0100 Subject: [PATCH 11/21] add comments --- src/ingester/parser/state_update.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ingester/parser/state_update.rs b/src/ingester/parser/state_update.rs index a9b5ae3b..1742a6bb 100644 --- a/src/ingester/parser/state_update.rs +++ b/src/ingester/parser/state_update.rs @@ -86,7 +86,9 @@ impl From for AddressQueueUpdate { #[derive(Default, Debug, Clone, PartialEq, Eq)] /// Representation of state update of the compression system that is optimal for simple persistence. pub struct StateUpdate { + // v1 and v2 tree accounts pub in_accounts: HashSet, + // v1 and v2 tree accounts pub out_accounts: Vec, pub account_transactions: HashSet, pub transactions: HashSet, From 9343e0c1021b6df187c7721d304007bd64338d72 Mon Sep 17 00:00:00 2001 From: ananas-block Date: Wed, 25 Jun 2025 15:33:48 +0100 Subject: [PATCH 12/21] added v1 inputs asserts --- .../persist_state_update_test.rs | 184 +++++++++++++++--- 1 file changed, 159 insertions(+), 25 deletions(-) diff --git a/tests/integration_tests/persist_state_update_test.rs b/tests/integration_tests/persist_state_update_test.rs index 2e64444a..c4b80ee5 100644 --- a/tests/integration_tests/persist_state_update_test.rs +++ b/tests/integration_tests/persist_state_update_test.rs @@ -56,7 +56,7 @@ impl CollectionConfig { #[derive(Debug, Clone)] pub struct StateUpdateConfig { // Collection configurations for StateUpdate fields - pub in_accounts: CollectionConfig, + pub in_accounts_v1: CollectionConfig, pub out_accounts_v1: CollectionConfig, pub out_accounts_v2: CollectionConfig, pub account_transactions: CollectionConfig, @@ -78,7 +78,7 @@ pub struct StateUpdateConfig { impl Default for StateUpdateConfig { fn default() -> Self { Self { - in_accounts: CollectionConfig::new(0, 5, 0.0), + in_accounts_v1: CollectionConfig::new(0, 3, 0.3), out_accounts_v1: CollectionConfig::new(0, 5, 1.0), out_accounts_v2: CollectionConfig::new(0, 5, 1.0), account_transactions: CollectionConfig::new(0, 3, 0.0), @@ -106,14 +106,26 @@ fn get_rnd_state_update( base_seq: u64, base_leaf_index_v1: u64, base_leaf_index_v2: u64, + v1_available_accounts_for_spending: &mut Vec, ) -> StateUpdate { let mut state_update = StateUpdate::default(); - // Generate in_accounts (HashSet) - if rng.gen_bool(config.in_accounts.probability) { - let count = rng.gen_range(config.in_accounts.min_entries..=config.in_accounts.max_entries); + // Generate in_accounts (HashSet) - v1 accounts that will be spent + if !v1_available_accounts_for_spending.is_empty() + && rng.gen_bool(config.in_accounts_v1.probability) + { + let max_to_spend = config + .in_accounts_v1 + .max_entries + .min(v1_available_accounts_for_spending.len()); + let count = rng.gen_range(config.in_accounts_v1.min_entries..=max_to_spend); + for _i in 0..count { - state_update.in_accounts.insert(Hash::new_unique()); + if !v1_available_accounts_for_spending.is_empty() { + let index = rng.gen_range(0..v1_available_accounts_for_spending.len()); + let account_hash = v1_available_accounts_for_spending.remove(index); + state_update.in_accounts.insert(account_hash); + } } } @@ -358,6 +370,32 @@ async fn persist_state_update_and_commit( Ok(()) } +/// Helper function to fetch pre-existing account models for input accounts +async fn fetch_pre_existing_input_models( + db_conn: &DatabaseConnection, + state_update: &StateUpdate, +) -> Result, Box> { + use photon_indexer::dao::generated::accounts; + use sea_orm::ColumnTrait; + + if state_update.in_accounts.is_empty() { + return Ok(Vec::new()); + } + + let input_hashes: Vec> = state_update + .in_accounts + .iter() + .map(|hash| hash.0.to_vec()) + .collect(); + + let models = accounts::Entity::find() + .filter(accounts::Column::Hash.is_in(input_hashes)) + .all(db_conn) + .await?; + + Ok(models) +} + /// Assert that all output accounts from the state update were inserted correctly into the database async fn assert_output_accounts_persisted( db_conn: &DatabaseConnection, @@ -447,6 +485,72 @@ async fn assert_output_accounts_persisted( Ok(()) } +/// Assert that all input accounts from the state update were marked as spent in the database +/// This function compares the complete account models, not just the spent flag +async fn assert_input_accounts_persisted( + db_conn: &DatabaseConnection, + state_update: &StateUpdate, + pre_existing_models: &[photon_indexer::dao::generated::accounts::Model], +) -> Result<(), Box> { + use photon_indexer::dao::generated::accounts; + use sea_orm::ColumnTrait; + + if state_update.in_accounts.is_empty() { + println!("✅ No input accounts - skipping input accounts verification"); + return Ok(()); + } + + // Create expected models from pre-existing models with spent=true, prev_spent=original spent + let mut expected_models: Vec = pre_existing_models + .iter() + .map(|model| { + accounts::Model { + spent: true, // Should be marked as spent + ..model.clone() // All other fields should remain the same + } + }) + .collect(); + + // Sort by hash for consistent comparison + expected_models.sort_by(|a, b| a.hash.cmp(&b.hash)); + + // Query database for accounts with matching hashes + let input_hashes: Vec> = state_update + .in_accounts + .iter() + .map(|hash| hash.0.to_vec()) + .collect(); + + let mut db_accounts = accounts::Entity::find() + .filter(accounts::Column::Hash.is_in(input_hashes)) + .all(db_conn) + .await?; + + // Sort by hash for consistent comparison + db_accounts.sort_by(|a, b| a.hash.cmp(&b.hash)); + + // Verify we found all input accounts + assert_eq!( + db_accounts.len(), + expected_models.len(), + "Expected {} input accounts in database, found {}", + expected_models.len(), + db_accounts.len() + ); + + // Single assert comparing the complete models + assert_eq!( + db_accounts, expected_models, + "Input accounts do not match expected complete models after spending" + ); + + println!( + "✅ Successfully verified {} input accounts were marked as spent with complete models", + db_accounts.len() + ); + Ok(()) +} + /// Assert that state tree root matches reference implementation after appending new hashes async fn assert_state_tree_root( db_conn: &DatabaseConnection, @@ -694,9 +798,9 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat let config = StateUpdateConfig::default(); // Verify config structure values - assert_eq!(config.in_accounts.min_entries, 0); - assert_eq!(config.in_accounts.max_entries, 5); - assert_eq!(config.in_accounts.probability, 0.0); + assert_eq!(config.in_accounts_v1.min_entries, 0); + assert_eq!(config.in_accounts_v1.max_entries, 3); + assert_eq!(config.in_accounts_v1.probability, 0.3); assert_eq!(config.out_accounts_v1.min_entries, 0); assert_eq!(config.out_accounts_v1.max_entries, 5); @@ -714,7 +818,8 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat let mut base_seq = 500; let mut base_leaf_index_v1 = 0; let mut base_leaf_index_v2 = 1000; // Use separate leaf index space for v2 - let num_iters = 10; + let mut v1_available_accounts_for_spending: Vec = Vec::new(); + let num_iters = 100; for slot in 0..num_iters { println!("iter {}", slot); let simple_state_update = get_rnd_state_update( @@ -724,9 +829,16 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat base_seq, base_leaf_index_v1, base_leaf_index_v2, + &mut v1_available_accounts_for_spending, ); println!("simple_state_update {:?}", simple_state_update); + // Fetch pre-existing account models for input accounts before persisting + let pre_existing_input_models = + fetch_pre_existing_input_models(setup.db_conn.as_ref(), &simple_state_update) + .await + .expect("Failed to fetch pre-existing input accounts"); + // Persist the simple state update let result = persist_state_update_and_commit(&setup.db_conn, simple_state_update.clone()).await; @@ -743,25 +855,47 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat .await .expect("Failed to verify output accounts persistence"); + // Assert that all input accounts were marked as spent with complete models + assert_input_accounts_persisted( + &setup.db_conn, + &simple_state_update, + &pre_existing_input_models, + ) + .await + .expect("Failed to verify input accounts persistence"); + // Assert that state tree root matches reference implementation assert_state_tree_root(&setup.db_conn, &mut reference_tree, &simple_state_update) .await .expect("Failed to verify state tree root"); - // Update leaf indices separately for v1 and v2 - let v1_count = simple_state_update - .out_accounts - .iter() - .filter(|acc| acc.context.tree_type == TreeType::StateV1 as u16) - .count() as u64; - let v2_count = simple_state_update - .out_accounts - .iter() - .filter(|acc| acc.context.tree_type == TreeType::StateV2 as u16) - .count() as u64; - - base_seq += simple_state_update.out_accounts.len() as u64; - base_leaf_index_v1 += v1_count; - base_leaf_index_v2 += v2_count; + + { + // Collect new v1 output accounts for future spending + let new_v1_accounts: Vec = simple_state_update + .out_accounts + .iter() + .filter(|acc| acc.context.tree_type == TreeType::StateV1 as u16) + .map(|acc| acc.account.hash.clone()) + .collect(); + v1_available_accounts_for_spending.extend(new_v1_accounts.iter().cloned()); + + // Update leaf indices separately for v1 and v2 + let v1_count = new_v1_accounts.len() as u64; + let v2_count = simple_state_update + .out_accounts + .iter() + .filter(|acc| acc.context.tree_type == TreeType::StateV2 as u16) + .count() as u64; + + base_seq += simple_state_update.out_accounts.len() as u64; + base_leaf_index_v1 += v1_count; + base_leaf_index_v2 += v2_count; + + println!( + "Available accounts for spending: {}", + v1_available_accounts_for_spending.len() + ); + } } println!("Config structure test completed successfully - unified CollectionConfig approach with incremental slot/seq/leaf_index working"); } From 40b68d30471ba9e1d8121b18c9db19f7ea5a17f7 Mon Sep 17 00:00:00 2001 From: ananas-block Date: Wed, 25 Jun 2025 16:02:49 +0100 Subject: [PATCH 13/21] cleanup and comments --- .../persist_state_update_test.rs | 321 ++++++++++++++---- 1 file changed, 264 insertions(+), 57 deletions(-) diff --git a/tests/integration_tests/persist_state_update_test.rs b/tests/integration_tests/persist_state_update_test.rs index c4b80ee5..e3c84290 100644 --- a/tests/integration_tests/persist_state_update_test.rs +++ b/tests/integration_tests/persist_state_update_test.rs @@ -52,11 +52,27 @@ impl CollectionConfig { } } +/// Metadata about what was generated in a StateUpdate +#[derive(Debug, Clone)] +pub struct StateUpdateMetadata { + pub in_accounts_v1_count: usize, + pub in_accounts_v2_count: usize, + pub out_accounts_v1_count: usize, + pub out_accounts_v2_count: usize, + pub account_transactions_count: usize, + pub transactions_count: usize, + pub leaf_nullifications_count: usize, + pub indexed_merkle_tree_updates_count: usize, + pub batch_nullify_context_count: usize, + pub batch_new_addresses_count: usize, +} + /// Configuration for generating random StateUpdate data #[derive(Debug, Clone)] pub struct StateUpdateConfig { // Collection configurations for StateUpdate fields pub in_accounts_v1: CollectionConfig, + pub in_accounts_v2: CollectionConfig, pub out_accounts_v1: CollectionConfig, pub out_accounts_v2: CollectionConfig, pub account_transactions: CollectionConfig, @@ -79,6 +95,7 @@ impl Default for StateUpdateConfig { fn default() -> Self { Self { in_accounts_v1: CollectionConfig::new(0, 3, 0.3), + in_accounts_v2: CollectionConfig::new(0, 3, 0.3), out_accounts_v1: CollectionConfig::new(0, 5, 1.0), out_accounts_v2: CollectionConfig::new(0, 5, 1.0), account_transactions: CollectionConfig::new(0, 3, 0.0), @@ -103,12 +120,26 @@ fn get_rnd_state_update( rng: &mut StdRng, config: &StateUpdateConfig, slot: u64, - base_seq: u64, + base_seq_v1: u64, base_leaf_index_v1: u64, base_leaf_index_v2: u64, + _base_nullifier_queue_index: u64, v1_available_accounts_for_spending: &mut Vec, -) -> StateUpdate { + v2_available_accounts_for_spending: &mut Vec, +) -> (StateUpdate, StateUpdateMetadata) { let mut state_update = StateUpdate::default(); + let mut metadata = StateUpdateMetadata { + in_accounts_v1_count: 0, + in_accounts_v2_count: 0, + out_accounts_v1_count: 0, + out_accounts_v2_count: 0, + account_transactions_count: 0, + transactions_count: 0, + leaf_nullifications_count: 0, + indexed_merkle_tree_updates_count: 0, + batch_nullify_context_count: 0, + batch_new_addresses_count: 0, + }; // Generate in_accounts (HashSet) - v1 accounts that will be spent if !v1_available_accounts_for_spending.is_empty() @@ -125,6 +156,27 @@ fn get_rnd_state_update( let index = rng.gen_range(0..v1_available_accounts_for_spending.len()); let account_hash = v1_available_accounts_for_spending.remove(index); state_update.in_accounts.insert(account_hash); + metadata.in_accounts_v1_count += 1; + } + } + } + + // Generate in_accounts (HashSet) - v2 accounts that will be spent + if !v2_available_accounts_for_spending.is_empty() + && rng.gen_bool(config.in_accounts_v2.probability) + { + let max_to_spend = config + .in_accounts_v2 + .max_entries + .min(v2_available_accounts_for_spending.len()); + let count = rng.gen_range(config.in_accounts_v2.min_entries..=max_to_spend); + + for _i in 0..count { + if !v2_available_accounts_for_spending.is_empty() { + let index = rng.gen_range(0..v2_available_accounts_for_spending.len()); + let account_hash = v2_available_accounts_for_spending.remove(index); + state_update.in_accounts.insert(account_hash); + metadata.in_accounts_v2_count += 1; } } } @@ -138,6 +190,7 @@ fn get_rnd_state_update( let count = rng.gen_range(config.out_accounts_v1.min_entries..=config.out_accounts_v1.max_entries); + metadata.out_accounts_v1_count = count as usize; for i in 0..count { let account = AccountWithContext { account: Account { @@ -166,7 +219,7 @@ fn get_rnd_state_update( ), tree: SerializablePubkey::from(test_tree_pubkey), leaf_index: UnsignedInteger(base_leaf_index_v1 + i as u64), - seq: Some(UnsignedInteger(base_seq + i as u64)), + seq: Some(UnsignedInteger(base_seq_v1 + i as u64)), slot_created: UnsignedInteger(slot), }, context: AccountContext { @@ -190,6 +243,7 @@ fn get_rnd_state_update( let count = rng.gen_range(config.out_accounts_v2.min_entries..=config.out_accounts_v2.max_entries); + metadata.out_accounts_v2_count = count as usize; for i in 0..count { let account = AccountWithContext { account: Account { @@ -291,7 +345,7 @@ fn get_rnd_state_update( state_update.leaf_nullifications.insert(LeafNullification { tree: test_tree_pubkey, leaf_index: base_leaf_index_v1 + i as u64, - seq: base_seq + i as u64, + seq: base_seq_v1 + i as u64, signature: Signature::from(sig_bytes), }); } @@ -315,7 +369,7 @@ fn get_rnd_state_update( index: (base_leaf_index_v1 + i as u64) as usize, }, hash: rng.gen::<[u8; 32]>(), - seq: base_seq + i as u64, + seq: base_seq_v1 + i as u64, }; state_update .indexed_merkle_tree_updates @@ -356,7 +410,7 @@ fn get_rnd_state_update( // Note: batch_merkle_tree_events is left as default since it's complex and rarely used - state_update + (state_update, metadata) } /// Helper function to persist a state update and commit the transaction @@ -396,14 +450,73 @@ async fn fetch_pre_existing_input_models( Ok(models) } +/// Helper function to update test state after processing a state update +fn update_test_state_after_iteration( + state_update: &StateUpdate, + metadata: &StateUpdateMetadata, + v1_available_accounts_for_spending: &mut Vec, + v2_available_accounts_for_spending: &mut Vec, + base_seq_v1: &mut u64, + base_leaf_index_v1: &mut u64, + base_leaf_index_v2: &mut u64, + base_nullifier_queue_index: &mut u64, +) { + // Collect new v1 output accounts for future spending + let new_v1_accounts: Vec = state_update + .out_accounts + .iter() + .filter(|acc| acc.context.tree_type == TreeType::StateV1 as u16) + .map(|acc| acc.account.hash.clone()) + .collect(); + v1_available_accounts_for_spending.extend(new_v1_accounts.iter().cloned()); + + // Collect new v2 output accounts for future spending + let new_v2_accounts: Vec = state_update + .out_accounts + .iter() + .filter(|acc| acc.context.tree_type == TreeType::StateV2 as u16) + .map(|acc| acc.account.hash.clone()) + .collect(); + v2_available_accounts_for_spending.extend(new_v2_accounts.iter().cloned()); + + // Update indices using metadata for precise counts + let v1_output_count = metadata.out_accounts_v1_count as u64; + let v2_output_count = metadata.out_accounts_v2_count as u64; + let _v1_input_count = metadata.in_accounts_v1_count as u64; + let v2_input_count = metadata.in_accounts_v2_count as u64; + + *base_seq_v1 += v1_output_count; + *base_leaf_index_v1 += v1_output_count; + *base_leaf_index_v2 += v2_output_count; + *base_nullifier_queue_index += v2_input_count; // Only v2 input accounts get nullifier queue positions + + println!( + "Available accounts for spending: v1={}, v2={}", + v1_available_accounts_for_spending.len(), + v2_available_accounts_for_spending.len() + ); +} + /// Assert that all output accounts from the state update were inserted correctly into the database async fn assert_output_accounts_persisted( db_conn: &DatabaseConnection, + metadata: &StateUpdateMetadata, state_update: &StateUpdate, ) -> Result<(), Box> { use photon_indexer::dao::generated::accounts; use sea_orm::ColumnTrait; + // Validate metadata matches actual state update + let expected_total_out_accounts = + metadata.out_accounts_v1_count + metadata.out_accounts_v2_count; + assert_eq!( + state_update.out_accounts.len(), + expected_total_out_accounts, + "Metadata out_accounts count ({}) doesn't match actual out_accounts ({})", + expected_total_out_accounts, + state_update.out_accounts.len() + ); + if state_update.out_accounts.is_empty() { // If no accounts expected, verify table is empty // let account_count = accounts::Entity::find().count(db_conn).await?; @@ -415,6 +528,30 @@ async fn assert_output_accounts_persisted( return Ok(()); } + // Validate v1/v2 split matches metadata + let actual_v1_count = state_update + .out_accounts + .iter() + .filter(|acc| acc.context.tree_type == 1) // TreeType::StateV1 + .count(); + let actual_v2_count = state_update + .out_accounts + .iter() + .filter(|acc| acc.context.tree_type == 3) // TreeType::StateV2 + .count(); + + assert_eq!( + actual_v1_count, metadata.out_accounts_v1_count, + "Metadata v1 out_accounts count ({}) doesn't match actual v1 count ({})", + metadata.out_accounts_v1_count, actual_v1_count + ); + + assert_eq!( + actual_v2_count, metadata.out_accounts_v2_count, + "Metadata v2 out_accounts count ({}) doesn't match actual v2 count ({})", + metadata.out_accounts_v2_count, actual_v2_count + ); + // Create expected models from state update let expected_models: Vec = state_update .out_accounts @@ -489,24 +626,72 @@ async fn assert_output_accounts_persisted( /// This function compares the complete account models, not just the spent flag async fn assert_input_accounts_persisted( db_conn: &DatabaseConnection, + metadata: &StateUpdateMetadata, state_update: &StateUpdate, pre_existing_models: &[photon_indexer::dao::generated::accounts::Model], + _base_nullifier_queue_index: u64, ) -> Result<(), Box> { use photon_indexer::dao::generated::accounts; use sea_orm::ColumnTrait; + // Validate metadata matches actual state update + let expected_total_in_accounts = metadata.in_accounts_v1_count + metadata.in_accounts_v2_count; + assert_eq!( + state_update.in_accounts.len(), + expected_total_in_accounts, + "Metadata in_accounts count ({}) doesn't match actual in_accounts ({})", + expected_total_in_accounts, + state_update.in_accounts.len() + ); + + // Validate we have the right number of pre-existing models + assert_eq!( + pre_existing_models.len(), + expected_total_in_accounts, + "Pre-existing models count ({}) doesn't match expected in_accounts ({})", + pre_existing_models.len(), + expected_total_in_accounts + ); + if state_update.in_accounts.is_empty() { println!("✅ No input accounts - skipping input accounts verification"); return Ok(()); } + // Validate v1/v2 split in pre-existing models matches metadata + let actual_v1_input_count = pre_existing_models + .iter() + .filter(|model| model.tree_type == Some(1)) // TreeType::StateV1 + .count(); + let actual_v2_input_count = pre_existing_models + .iter() + .filter(|model| model.tree_type == Some(3)) // TreeType::StateV2 + .count(); + + assert_eq!( + actual_v1_input_count, metadata.in_accounts_v1_count, + "Metadata v1 in_accounts count ({}) doesn't match actual v1 pre-existing count ({})", + metadata.in_accounts_v1_count, actual_v1_input_count + ); + + assert_eq!( + actual_v2_input_count, metadata.in_accounts_v2_count, + "Metadata v2 in_accounts count ({}) doesn't match actual v2 pre-existing count ({})", + metadata.in_accounts_v2_count, actual_v2_input_count + ); + // Create expected models from pre-existing models with spent=true, prev_spent=original spent let mut expected_models: Vec = pre_existing_models .iter() .map(|model| { + // For all accounts (v1 and v2), spend_input_accounts() sets: + // - spent: true + // - prev_spent: Some(original_spent) + // v2-specific fields (nullifier_queue_index, tx_hash) are handled by BatchNullifyContext accounts::Model { - spent: true, // Should be marked as spent - ..model.clone() // All other fields should remain the same + spent: true, // Should be marked as spent + prev_spent: Some(model.spent), // prev_spent should be the original spent value + ..model.clone() // All other fields should remain the same } }) .collect(); @@ -554,11 +739,24 @@ async fn assert_input_accounts_persisted( /// Assert that state tree root matches reference implementation after appending new hashes async fn assert_state_tree_root( db_conn: &DatabaseConnection, - reference_tree: &mut MerkleTree, + metadata: &StateUpdateMetadata, state_update: &StateUpdate, + v1_reference_tree: &mut MerkleTree, ) -> Result<(), Box> { use photon_indexer::dao::generated::state_trees; use sea_orm::ColumnTrait; + + // Validate metadata consistency (same as output accounts validation) + let expected_total_out_accounts = + metadata.out_accounts_v1_count + metadata.out_accounts_v2_count; + assert_eq!( + state_update.out_accounts.len(), + expected_total_out_accounts, + "State tree: Metadata out_accounts count ({}) doesn't match actual out_accounts ({})", + expected_total_out_accounts, + state_update.out_accounts.len() + ); + if state_update.out_accounts.is_empty() { println!("✅ No output accounts - skipping state tree root verification"); return Ok(()); @@ -672,11 +870,11 @@ async fn assert_state_tree_root( for account_with_context in &v1_accounts { let leaf_hash = account_with_context.account.hash.0; - reference_tree.append(&leaf_hash)?; + v1_reference_tree.append(&leaf_hash)?; } // Get reference tree root after construction - let reference_root = reference_tree.root(); + let reference_root = v1_reference_tree.root(); println!("Reference tree root: {}", hex::encode(&reference_root)); // Get database root node for comparison @@ -792,7 +990,7 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat println!("\n\nconfig structure test seed {}\n\n", seed); let mut rng = StdRng::seed_from_u64(seed); - let mut reference_tree = MerkleTree::::new(26, 0); + let mut v1_reference_tree = MerkleTree::::new(26, 0); // Test that the new config structure works correctly let config = StateUpdateConfig::default(); @@ -802,6 +1000,10 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat assert_eq!(config.in_accounts_v1.max_entries, 3); assert_eq!(config.in_accounts_v1.probability, 0.3); + assert_eq!(config.in_accounts_v2.min_entries, 0); + assert_eq!(config.in_accounts_v2.max_entries, 3); + assert_eq!(config.in_accounts_v2.probability, 0.3); + assert_eq!(config.out_accounts_v1.min_entries, 0); assert_eq!(config.out_accounts_v1.max_entries, 5); assert_eq!(config.out_accounts_v1.probability, 1.0); @@ -815,87 +1017,92 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat assert_eq!(config.transactions.probability, 0.0); // Test that we can create a state update with incremental values - let mut base_seq = 500; + let mut base_seq_v1 = 500; let mut base_leaf_index_v1 = 0; let mut base_leaf_index_v2 = 1000; // Use separate leaf index space for v2 + let mut base_nullifier_queue_index = 0; // Track nullifier queue position for v2 input accounts let mut v1_available_accounts_for_spending: Vec = Vec::new(); + let mut v2_available_accounts_for_spending: Vec = Vec::new(); let num_iters = 100; + + // Steps: + // 1. Generate random state update + // 2. Fetch pre-existing account models for input accounts before persisting + // 3. Persist the simple state update + // 4. Assert output accounts + // 5. Assert input accounts + // 6. Assert state tree root matches reference tree root + // 7. Update test state for slot in 0..num_iters { println!("iter {}", slot); - let simple_state_update = get_rnd_state_update( + // 1. Generate random state update + let (state_update, metadata) = get_rnd_state_update( &mut rng, &config, slot, - base_seq, + base_seq_v1, base_leaf_index_v1, base_leaf_index_v2, + base_nullifier_queue_index, &mut v1_available_accounts_for_spending, + &mut v2_available_accounts_for_spending, ); - println!("simple_state_update {:?}", simple_state_update); + println!("state_update {:?}", state_update); - // Fetch pre-existing account models for input accounts before persisting + // 2. Fetch pre-existing account models for input accounts before persisting let pre_existing_input_models = - fetch_pre_existing_input_models(setup.db_conn.as_ref(), &simple_state_update) + fetch_pre_existing_input_models(setup.db_conn.as_ref(), &state_update) .await .expect("Failed to fetch pre-existing input accounts"); - // Persist the simple state update - let result = - persist_state_update_and_commit(&setup.db_conn, simple_state_update.clone()).await; + // 3. Persist the random state update + let result = persist_state_update_and_commit(&setup.db_conn, state_update.clone()).await; // Should complete successfully assert!( result.is_ok(), - "Failed to persist simple state update: {:?}", + "Failed to persist random state update: {:?}", result.err() ); - // Assert that all output accounts were persisted correctly - assert_output_accounts_persisted(&setup.db_conn, &simple_state_update) + // 4. Assert that all output accounts were persisted correctly + assert_output_accounts_persisted(&setup.db_conn, &metadata, &state_update) .await .expect("Failed to verify output accounts persistence"); - // Assert that all input accounts were marked as spent with complete models + // 5. Assert that all input accounts were marked as spent with complete models assert_input_accounts_persisted( &setup.db_conn, - &simple_state_update, + &metadata, + &state_update, &pre_existing_input_models, + base_nullifier_queue_index, ) .await .expect("Failed to verify input accounts persistence"); - // Assert that state tree root matches reference implementation - assert_state_tree_root(&setup.db_conn, &mut reference_tree, &simple_state_update) - .await - .expect("Failed to verify state tree root"); - - { - // Collect new v1 output accounts for future spending - let new_v1_accounts: Vec = simple_state_update - .out_accounts - .iter() - .filter(|acc| acc.context.tree_type == TreeType::StateV1 as u16) - .map(|acc| acc.account.hash.clone()) - .collect(); - v1_available_accounts_for_spending.extend(new_v1_accounts.iter().cloned()); - - // Update leaf indices separately for v1 and v2 - let v1_count = new_v1_accounts.len() as u64; - let v2_count = simple_state_update - .out_accounts - .iter() - .filter(|acc| acc.context.tree_type == TreeType::StateV2 as u16) - .count() as u64; - - base_seq += simple_state_update.out_accounts.len() as u64; - base_leaf_index_v1 += v1_count; - base_leaf_index_v2 += v2_count; + // 6. Assert that state tree root matches reference tree root + // - updates reference tree + assert_state_tree_root( + &setup.db_conn, + &metadata, + &state_update, + &mut v1_reference_tree, + ) + .await + .expect("Failed to verify state tree root"); - println!( - "Available accounts for spending: {}", - v1_available_accounts_for_spending.len() - ); - } + // 7. Update test state after processing the state update + update_test_state_after_iteration( + &state_update, + &metadata, + &mut v1_available_accounts_for_spending, + &mut v2_available_accounts_for_spending, + &mut base_seq_v1, + &mut base_leaf_index_v1, + &mut base_leaf_index_v2, + &mut base_nullifier_queue_index, + ); } println!("Config structure test completed successfully - unified CollectionConfig approach with incremental slot/seq/leaf_index working"); } From 274ca7051dec5aa4eca69521f3dbfe56d884fe81 Mon Sep 17 00:00:00 2001 From: ananas-block Date: Wed, 25 Jun 2025 17:48:46 +0100 Subject: [PATCH 14/21] stash --- src/ingester/parser/state_update.rs | 3 +- .../persist_state_update_test.rs | 57 +++++++++++-------- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/src/ingester/parser/state_update.rs b/src/ingester/parser/state_update.rs index 1742a6bb..549a2284 100644 --- a/src/ingester/parser/state_update.rs +++ b/src/ingester/parser/state_update.rs @@ -94,8 +94,9 @@ pub struct StateUpdate { pub transactions: HashSet, pub leaf_nullifications: HashSet, pub indexed_merkle_tree_updates: HashMap<(Pubkey, u64), IndexedTreeLeafUpdate>, - + // v2 state and address Merkle tree updates pub batch_merkle_tree_events: BatchMerkleTreeEvents, + // v2 input accounts that are inserted into the input queue pub batch_nullify_context: Vec, pub batch_new_addresses: Vec, } diff --git a/tests/integration_tests/persist_state_update_test.rs b/tests/integration_tests/persist_state_update_test.rs index e3c84290..c062708d 100644 --- a/tests/integration_tests/persist_state_update_test.rs +++ b/tests/integration_tests/persist_state_update_test.rs @@ -123,7 +123,7 @@ fn get_rnd_state_update( base_seq_v1: u64, base_leaf_index_v1: u64, base_leaf_index_v2: u64, - _base_nullifier_queue_index: u64, + base_nullifier_queue_index: u64, v1_available_accounts_for_spending: &mut Vec, v2_available_accounts_for_spending: &mut Vec, ) -> (StateUpdate, StateUpdateMetadata) { @@ -377,21 +377,20 @@ fn get_rnd_state_update( } } - // Generate batch_nullify_context (Vec) - if rng.gen_bool(config.batch_nullify_context.probability) { - let count = rng.gen_range( - config.batch_nullify_context.min_entries..=config.batch_nullify_context.max_entries, - ); - for i in 0..count { - state_update - .batch_nullify_context - .push(BatchNullifyContext { - tx_hash: rng.gen::<[u8; 32]>(), - account_hash: rng.gen::<[u8; 32]>(), - nullifier: rng.gen::<[u8; 32]>(), - nullifier_queue_index: i as u64, - }); - } + // Generate batch_nullify_context (Vec) for actual input accounts + // Create BatchNullifyContext entries for each input account so they get their v2-specific fields set + let mut nullifier_queue_index = base_nullifier_queue_index; + for account_hash in &state_update.in_accounts { + state_update + .batch_nullify_context + .push(BatchNullifyContext { + tx_hash: rng.gen::<[u8; 32]>(), + account_hash: account_hash.0, // Use the actual input account hash + nullifier: rng.gen::<[u8; 32]>(), + nullifier_queue_index, + }); + nullifier_queue_index += 1; + metadata.batch_nullify_context_count += 1; } // Generate batch_new_addresses (Vec) @@ -684,15 +683,25 @@ async fn assert_input_accounts_persisted( let mut expected_models: Vec = pre_existing_models .iter() .map(|model| { - // For all accounts (v1 and v2), spend_input_accounts() sets: - // - spent: true - // - prev_spent: Some(original_spent) - // v2-specific fields (nullifier_queue_index, tx_hash) are handled by BatchNullifyContext - accounts::Model { - spent: true, // Should be marked as spent - prev_spent: Some(model.spent), // prev_spent should be the original spent value - ..model.clone() // All other fields should remain the same + let mut updated_model = accounts::Model { + spent: true, // Should be marked as spent + prev_spent: Some(false), // prev_spent should be the original spent value + ..model.clone() // All other fields should remain the same + }; + + // For accounts that have BatchNullifyContext, set the v2-specific fields from the state update + if let Some(batch_context) = state_update + .batch_nullify_context + .iter() + .find(|ctx| ctx.account_hash.as_slice() == model.hash.as_slice()) + { + updated_model.nullifier_queue_index = + Some(batch_context.nullifier_queue_index as i64); + updated_model.nullifier = Some(batch_context.nullifier.to_vec()); + updated_model.tx_hash = Some(batch_context.tx_hash.to_vec()); } + + updated_model }) .collect(); From fecfeb434c628ba31817bfdb4bc5f86c1f97d3c8 Mon Sep 17 00:00:00 2001 From: ananas-block Date: Wed, 25 Jun 2025 18:43:31 +0100 Subject: [PATCH 15/21] chore: add indexed Merkle tree to rnd test --- Cargo.lock | 18 ++ Cargo.toml | 1 + .../persist_state_update_test.rs | 196 +++++++++++++++++- 3 files changed, 204 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ae825d0c..bfd7fe6b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3769,6 +3769,7 @@ dependencies = [ "light-bounded-vec", "light-hasher", "memoffset 0.9.1", + "solana-program-error", "thiserror 2.0.12", ] @@ -3804,6 +3805,22 @@ dependencies = [ "thiserror 2.0.12", ] +[[package]] +name = "light-indexed-merkle-tree" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f38362948ad7b8ae1fd1626d38743bed5a15563336fb5d4148b9162186c8e55" +dependencies = [ + "light-bounded-vec", + "light-concurrent-merkle-tree", + "light-hasher", + "light-merkle-tree-reference", + "num-bigint 0.4.6", + "num-traits", + "solana-program-error", + "thiserror 2.0.12", +] + [[package]] name = "light-macros" version = "2.1.0" @@ -4565,6 +4582,7 @@ dependencies = [ "light-compressed-account", "light-concurrent-merkle-tree", "light-hasher", + "light-indexed-merkle-tree", "light-merkle-tree-metadata", "light-merkle-tree-reference", "light-poseidon 0.3.0", diff --git a/Cargo.toml b/Cargo.toml index e46e5ea0..ab314a47 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -132,6 +132,7 @@ rust-s3 = "0.34.0" function_name = "0.3.0" serial_test = "2.0.0" light-merkle-tree-reference = "2.0.0" +light-indexed-merkle-tree = "2.1.0" light-hasher = "3.1.0" [profile.dev] diff --git a/tests/integration_tests/persist_state_update_test.rs b/tests/integration_tests/persist_state_update_test.rs index c062708d..f80775d7 100644 --- a/tests/integration_tests/persist_state_update_test.rs +++ b/tests/integration_tests/persist_state_update_test.rs @@ -3,7 +3,12 @@ use function_name::named; use light_compressed_account::indexer_event::event::BatchNullifyContext; use light_compressed_account::TreeType; use light_hasher::Poseidon; +use light_indexed_merkle_tree::{ + array::{IndexedArray, IndexedElement}, + reference::IndexedMerkleTree, +}; use light_merkle_tree_reference::MerkleTree; +use num_bigint::BigUint; use photon_indexer::common::typedefs::account::AccountData; use photon_indexer::common::typedefs::account::{Account, AccountContext, AccountWithContext}; use photon_indexer::common::typedefs::bs64_string::Base64String; @@ -101,7 +106,7 @@ impl Default for StateUpdateConfig { account_transactions: CollectionConfig::new(0, 3, 0.0), transactions: CollectionConfig::new(0, 2, 0.0), leaf_nullifications: CollectionConfig::new(0, 3, 0.0), - indexed_merkle_tree_updates: CollectionConfig::new(0, 3, 0.0), + indexed_merkle_tree_updates: CollectionConfig::new(0, 3, 1.0), batch_nullify_context: CollectionConfig::new(0, 2, 0.0), batch_new_addresses: CollectionConfig::new(0, 3, 0.0), @@ -124,6 +129,7 @@ fn get_rnd_state_update( base_leaf_index_v1: u64, base_leaf_index_v2: u64, base_nullifier_queue_index: u64, + base_indexed_seq: u64, v1_available_accounts_for_spending: &mut Vec, v2_available_accounts_for_spending: &mut Vec, ) -> (StateUpdate, StateUpdateMetadata) { @@ -357,23 +363,51 @@ fn get_rnd_state_update( config.indexed_merkle_tree_updates.min_entries ..=config.indexed_merkle_tree_updates.max_entries, ); + metadata.indexed_merkle_tree_updates_count = count; for i in 0..count { let tree = test_tree_pubkey; - let index = base_leaf_index_v1 + i as u64; + let leaf_index = base_indexed_seq + i as u64; + + // Generate simple but unique values for indexed elements + let mut value_bytes = [0u8; 32]; + value_bytes[31] = (leaf_index % 256) as u8; // Make each value unique + + let mut next_value_bytes = [0u8; 32]; + next_value_bytes[31] = 0; + + // For next_index, use a reasonable sequential value + let next_index = if i == count - 1 { + 0 + } else { + (base_indexed_seq + i as u64 + 1) as usize + }; + + // Compute the proper hash for the indexed element using the IndexedElement::hash method + let indexed_element = IndexedElement { + index: leaf_index as usize, + value: BigUint::from_bytes_be(&value_bytes), + next_index, + }; + let next_value = BigUint::from_bytes_be(&next_value_bytes); + let hash = indexed_element + .hash::(&next_value) + .map_err(|e| format!("Failed to compute indexed element hash: {}", e)) + .unwrap(); + let update = IndexedTreeLeafUpdate { tree, leaf: RawIndexedElement { - value: rng.gen::<[u8; 32]>(), - next_index: rng.gen::() as usize, - next_value: rng.gen::<[u8; 32]>(), - index: (base_leaf_index_v1 + i as u64) as usize, + value: value_bytes, + next_index, + next_value: next_value_bytes, + index: leaf_index as usize, }, - hash: rng.gen::<[u8; 32]>(), - seq: base_seq_v1 + i as u64, + hash, + seq: base_indexed_seq + i as u64, }; state_update .indexed_merkle_tree_updates - .insert((tree, index), update); + .insert((tree, leaf_index), update); } } @@ -459,6 +493,7 @@ fn update_test_state_after_iteration( base_leaf_index_v1: &mut u64, base_leaf_index_v2: &mut u64, base_nullifier_queue_index: &mut u64, + base_indexed_seq: &mut u64, ) { // Collect new v1 output accounts for future spending let new_v1_accounts: Vec = state_update @@ -483,11 +518,13 @@ fn update_test_state_after_iteration( let v2_output_count = metadata.out_accounts_v2_count as u64; let _v1_input_count = metadata.in_accounts_v1_count as u64; let v2_input_count = metadata.in_accounts_v2_count as u64; + let indexed_updates_count = metadata.indexed_merkle_tree_updates_count as u64; *base_seq_v1 += v1_output_count; *base_leaf_index_v1 += v1_output_count; *base_leaf_index_v2 += v2_output_count; *base_nullifier_queue_index += v2_input_count; // Only v2 input accounts get nullifier queue positions + *base_indexed_seq += indexed_updates_count; // Track indexed tree sequence println!( "Available accounts for spending: v1={}, v2={}", @@ -923,6 +960,123 @@ async fn assert_state_tree_root( Ok(()) } +/// Assert that indexed tree root matches reference implementation after applying updates +async fn assert_indexed_tree_root( + db_conn: &DatabaseConnection, + metadata: &StateUpdateMetadata, + state_update: &StateUpdate, + reference_indexed_tree: &mut IndexedMerkleTree, + reference_indexed_array: &mut IndexedArray, +) -> Result<(), Box> { + use photon_indexer::dao::generated::indexed_trees; + use sea_orm::ColumnTrait; + + // Validate metadata consistency + assert_eq!( + state_update.indexed_merkle_tree_updates.len(), + metadata.indexed_merkle_tree_updates_count, + "Metadata indexed_merkle_tree_updates count ({}) doesn't match actual updates ({})", + metadata.indexed_merkle_tree_updates_count, + state_update.indexed_merkle_tree_updates.len() + ); + + if state_update.indexed_merkle_tree_updates.is_empty() { + println!("✅ No indexed tree updates - skipping indexed tree root verification"); + return Ok(()); + } + + // Get the tree pubkey from the first indexed tree update + let first_update = state_update + .indexed_merkle_tree_updates + .values() + .next() + .unwrap(); + let tree_pubkey_bytes = first_update.tree.to_bytes().to_vec(); + + println!("Indexed Tree Updates:"); + for ((tree, leaf_index), update) in &state_update.indexed_merkle_tree_updates { + println!( + " Update at leaf_index {} for tree {}: value={}, next_index={}, seq={}", + leaf_index, + hex::encode(tree.to_bytes()), + hex::encode(update.leaf.value), + update.leaf.next_index, + update.seq + ); + } + + // Apply updates to reference tree + for ((_, _), update) in &state_update.indexed_merkle_tree_updates { + let value = BigUint::from_bytes_be(&update.leaf.value); + println!("Appending value {} to reference indexed tree", value); + + // Use the reference tree append method + reference_indexed_tree.append(&value, reference_indexed_array)?; + } + + // Get reference tree root + let reference_root = reference_indexed_tree.root(); + println!( + "Reference indexed tree root: {}", + hex::encode(&reference_root) + ); + + // Query database for indexed tree entries + let db_indexed_entries = indexed_trees::Entity::find() + .filter(indexed_trees::Column::Tree.eq(tree_pubkey_bytes.clone())) + .all(db_conn) + .await?; + + println!("Database Indexed Tree Entries:"); + for entry in &db_indexed_entries { + println!( + " leaf_index={}, value={}, next_index={}, next_value={}, seq={:?}", + entry.leaf_index, + hex::encode(&entry.value), + entry.next_index, + hex::encode(&entry.next_value), + entry.seq + ); + } + + // For now, let's just verify that our updates were persisted correctly + // Full root verification would require rebuilding the entire indexed tree from database state + + // Verify that we have entries for each update we made + for ((_, leaf_index), update) in &state_update.indexed_merkle_tree_updates { + let found_entry = db_indexed_entries + .iter() + .find(|entry| entry.leaf_index == *leaf_index as i64); + + assert!( + found_entry.is_some(), + "Indexed tree update at leaf_index {} not found in database", + leaf_index + ); + + let entry = found_entry.unwrap(); + assert_eq!( + entry.value, + update.leaf.value.to_vec(), + "Database value doesn't match update at leaf_index {}", + leaf_index + ); + assert_eq!( + entry.next_index, update.leaf.next_index as i64, + "Database next_index doesn't match update at leaf_index {}", + leaf_index + ); + } + + println!("✅ Indexed tree updates verification successful!"); + println!( + "Reference tree has {} entries", + reference_indexed_array.len() + ); + + Ok(()) +} + #[named] #[rstest] #[tokio::test] @@ -1001,6 +1155,11 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat let mut v1_reference_tree = MerkleTree::::new(26, 0); + // Initialize reference indexed tree (v1 style) + let mut reference_indexed_tree = IndexedMerkleTree::::new(26, 10).unwrap(); + let mut reference_indexed_array = IndexedArray::::default(); + reference_indexed_tree.init().unwrap(); + // Test that the new config structure works correctly let config = StateUpdateConfig::default(); @@ -1030,9 +1189,10 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat let mut base_leaf_index_v1 = 0; let mut base_leaf_index_v2 = 1000; // Use separate leaf index space for v2 let mut base_nullifier_queue_index = 0; // Track nullifier queue position for v2 input accounts + let mut base_indexed_seq = 2; // Start at 2 since indexed tree is initialized with 0 and 1 let mut v1_available_accounts_for_spending: Vec = Vec::new(); let mut v2_available_accounts_for_spending: Vec = Vec::new(); - let num_iters = 100; + let num_iters = 1; // Steps: // 1. Generate random state update @@ -1053,6 +1213,7 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat base_leaf_index_v1, base_leaf_index_v2, base_nullifier_queue_index, + base_indexed_seq, &mut v1_available_accounts_for_spending, &mut v2_available_accounts_for_spending, ); @@ -1101,7 +1262,19 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat .await .expect("Failed to verify state tree root"); - // 7. Update test state after processing the state update + // 7. Assert that indexed tree updates were persisted correctly + // - updates reference indexed tree + assert_indexed_tree_root( + &setup.db_conn, + &metadata, + &state_update, + &mut reference_indexed_tree, + &mut reference_indexed_array, + ) + .await + .expect("Failed to verify indexed tree root"); + + // 8. Update test state after processing the state update update_test_state_after_iteration( &state_update, &metadata, @@ -1111,6 +1284,7 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat &mut base_leaf_index_v1, &mut base_leaf_index_v2, &mut base_nullifier_queue_index, + &mut base_indexed_seq, ); } println!("Config structure test completed successfully - unified CollectionConfig approach with incremental slot/seq/leaf_index working"); From 0a319059dc689b7b30d403ff7dcb2d819b58e39f Mon Sep 17 00:00:00 2001 From: ananas-block Date: Wed, 25 Jun 2025 18:45:09 +0100 Subject: [PATCH 16/21] hardcoded see to reproduce --- tests/integration_tests/persist_state_update_test.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/persist_state_update_test.rs b/tests/integration_tests/persist_state_update_test.rs index f80775d7..ce8125b9 100644 --- a/tests/integration_tests/persist_state_update_test.rs +++ b/tests/integration_tests/persist_state_update_test.rs @@ -1149,7 +1149,7 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat // Set up deterministic randomness following the light-protocol pattern let mut thread_rng = ThreadRng::default(); let random_seed = thread_rng.next_u64(); - let seed: u64 = random_seed; + let seed: u64 = 5331180609400104349; //random_seed; println!("\n\nconfig structure test seed {}\n\n", seed); let mut rng = StdRng::seed_from_u64(seed); @@ -1192,7 +1192,7 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat let mut base_indexed_seq = 2; // Start at 2 since indexed tree is initialized with 0 and 1 let mut v1_available_accounts_for_spending: Vec = Vec::new(); let mut v2_available_accounts_for_spending: Vec = Vec::new(); - let num_iters = 1; + let num_iters = 100; // Steps: // 1. Generate random state update From 3aba5a57c9899c4ffcbd8656503c9209d861a7ea Mon Sep 17 00:00:00 2001 From: ananas-block Date: Wed, 25 Jun 2025 20:09:48 +0100 Subject: [PATCH 17/21] add leaf asserts --- .../persist_state_update_test.rs | 185 ++++++++++++------ 1 file changed, 127 insertions(+), 58 deletions(-) diff --git a/tests/integration_tests/persist_state_update_test.rs b/tests/integration_tests/persist_state_update_test.rs index ce8125b9..78f566bd 100644 --- a/tests/integration_tests/persist_state_update_test.rs +++ b/tests/integration_tests/persist_state_update_test.rs @@ -2,11 +2,9 @@ use crate::utils::*; use function_name::named; use light_compressed_account::indexer_event::event::BatchNullifyContext; use light_compressed_account::TreeType; +use light_hasher::bigint::bigint_to_be_bytes_array; use light_hasher::Poseidon; -use light_indexed_merkle_tree::{ - array::{IndexedArray, IndexedElement}, - reference::IndexedMerkleTree, -}; +use light_indexed_merkle_tree::{array::IndexedArray, reference::IndexedMerkleTree}; use light_merkle_tree_reference::MerkleTree; use num_bigint::BigUint; use photon_indexer::common::typedefs::account::AccountData; @@ -132,6 +130,8 @@ fn get_rnd_state_update( base_indexed_seq: u64, v1_available_accounts_for_spending: &mut Vec, v2_available_accounts_for_spending: &mut Vec, + indexed_array: &mut IndexedArray, + reference_indexed_tree: &mut IndexedMerkleTree, ) -> (StateUpdate, StateUpdateMetadata) { let mut state_update = StateUpdate::default(); let mut metadata = StateUpdateMetadata { @@ -365,49 +365,63 @@ fn get_rnd_state_update( ); metadata.indexed_merkle_tree_updates_count = count; for i in 0..count { - let tree = test_tree_pubkey; - let leaf_index = base_indexed_seq + i as u64; + let tree = solana_pubkey::pubkey!("amt1Ayt45jfbdw5YSo7iz6WZxUmnZsQTYXy82hVwyC2"); // Generate simple but unique values for indexed elements - let mut value_bytes = [0u8; 32]; - value_bytes[31] = (leaf_index % 256) as u8; // Make each value unique - - let mut next_value_bytes = [0u8; 32]; - next_value_bytes[31] = 0; - - // For next_index, use a reasonable sequential value - let next_index = if i == count - 1 { - 0 - } else { - (base_indexed_seq + i as u64 + 1) as usize - }; - - // Compute the proper hash for the indexed element using the IndexedElement::hash method - let indexed_element = IndexedElement { - index: leaf_index as usize, - value: BigUint::from_bytes_be(&value_bytes), - next_index, - }; - let next_value = BigUint::from_bytes_be(&next_value_bytes); - let hash = indexed_element - .hash::(&next_value) - .map_err(|e| format!("Failed to compute indexed element hash: {}", e)) + let mut value_bytes = rng.gen::<[u8; 32]>(); + value_bytes[0] = 0; + let value = BigUint::from_bytes_be(&value_bytes); + + // // Find the low element for this value using the indexed array + // let (old_low_element, old_low_element_next_value) = indexed_array + // .find_low_element_for_nonexistent(&value) + // .unwrap(); + + // // Create the nullifier bundle which contains all the proper elements + // let nullifier_bundle = indexed_array + // .new_element_with_low_element_index(old_low_element.index, &value) + // .unwrap(); + + // // Use the bundle data to create the update + // let leaf_index = nullifier_bundle.new_element.index; + // let next_index = nullifier_bundle.new_element.next_index; + // let next_value_bytes = + // bigint_to_be_bytes_array::<32>(&nullifier_bundle.new_element_next_value).unwrap(); + + // // Compute the proper hash for the indexed element + // let hash = nullifier_bundle + // .new_element + // .hash::(&nullifier_bundle.new_element_next_value) + // .unwrap(); + let nullifier_bundle = indexed_array.append(&value).unwrap(); + let hash = nullifier_bundle + .new_element + .hash::(&nullifier_bundle.new_element_next_value) + .unwrap(); + reference_indexed_tree + .update( + &nullifier_bundle.new_low_element, + &nullifier_bundle.new_element, + &nullifier_bundle.new_element_next_value, + ) .unwrap(); - let update = IndexedTreeLeafUpdate { tree, leaf: RawIndexedElement { value: value_bytes, - next_index, - next_value: next_value_bytes, - index: leaf_index as usize, + next_index: nullifier_bundle.new_element.next_index, + next_value: bigint_to_be_bytes_array::<32>( + &nullifier_bundle.new_element_next_value, + ) + .unwrap(), + index: nullifier_bundle.new_element.index, }, hash, seq: base_indexed_seq + i as u64, }; state_update .indexed_merkle_tree_updates - .insert((tree, leaf_index), update); + .insert((tree, nullifier_bundle.new_element.index as u64), update); } } @@ -965,8 +979,8 @@ async fn assert_indexed_tree_root( db_conn: &DatabaseConnection, metadata: &StateUpdateMetadata, state_update: &StateUpdate, - reference_indexed_tree: &mut IndexedMerkleTree, - reference_indexed_array: &mut IndexedArray, + reference_root: [u8; 32], + reference_tree: &IndexedMerkleTree, ) -> Result<(), Box> { use photon_indexer::dao::generated::indexed_trees; use sea_orm::ColumnTrait; @@ -1005,28 +1019,43 @@ async fn assert_indexed_tree_root( ); } - // Apply updates to reference tree - for ((_, _), update) in &state_update.indexed_merkle_tree_updates { - let value = BigUint::from_bytes_be(&update.leaf.value); - println!("Appending value {} to reference indexed tree", value); - - // Use the reference tree append method - reference_indexed_tree.append(&value, reference_indexed_array)?; - } - - // Get reference tree root - let reference_root = reference_indexed_tree.root(); - println!( - "Reference indexed tree root: {}", - hex::encode(&reference_root) - ); - // Query database for indexed tree entries let db_indexed_entries = indexed_trees::Entity::find() .filter(indexed_trees::Column::Tree.eq(tree_pubkey_bytes.clone())) .all(db_conn) .await?; + // Get database root from state_trees table (indexed trees use same table structure) + use photon_indexer::dao::generated::state_trees; + + let ref_leaves = (0..4) + .map(|i| reference_tree.merkle_tree.get_leaf(i).unwrap()) + .collect::>(); + let mut leaf_nodes_from_db = state_trees::Entity::find() + .filter(state_trees::Column::Tree.eq(tree_pubkey_bytes.clone())) + .filter(state_trees::Column::Level.eq(0)) + .all(db_conn) + .await?; + + // Sort by leaf index + leaf_nodes_from_db.sort_by_key(|node| node.leaf_idx.unwrap_or(0)); + + let leaf_nodes = leaf_nodes_from_db + .iter() + .map(|x| { + let mut hash_array = [0u8; 32]; + hash_array.copy_from_slice(&x.hash); + hash_array + }) + .collect::>(); + assert_eq!(leaf_nodes, ref_leaves); + + let root_nodes = state_trees::Entity::find() + .filter(state_trees::Column::Tree.eq(tree_pubkey_bytes.clone())) + .filter(state_trees::Column::Level.eq(26)) + .all(db_conn) + .await?; + println!("Database Indexed Tree Entries:"); for entry in &db_indexed_entries { println!( @@ -1069,10 +1098,38 @@ async fn assert_indexed_tree_root( } println!("✅ Indexed tree updates verification successful!"); - println!( - "Reference tree has {} entries", - reference_indexed_array.len() - ); + + // Verify root nodes + if root_nodes.len() == 1 { + let root_node = &root_nodes[0]; + let mut db_root_array = [0u8; 32]; + db_root_array.copy_from_slice(&root_node.hash); + + println!( + "Reference indexed tree root: {}", + hex::encode(&reference_root) + ); + println!( + "Database indexed tree root: {}", + hex::encode(&db_root_array) + ); + + assert_eq!( + reference_root, + db_root_array, + "Indexed tree root mismatch!\nReference: {}\nDatabase: {}", + hex::encode(&reference_root), + hex::encode(&db_root_array) + ); + + println!("✅ Indexed tree root verification successful!"); + } else { + return Err(format!( + "Expected exactly 1 root node for indexed tree, found {}", + root_nodes.len() + ) + .into()); + } Ok(()) } @@ -1158,7 +1215,17 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat // Initialize reference indexed tree (v1 style) let mut reference_indexed_tree = IndexedMerkleTree::::new(26, 10).unwrap(); let mut reference_indexed_array = IndexedArray::::default(); + reference_indexed_array.init().unwrap(); reference_indexed_tree.init().unwrap(); + // let height = 26; + // let canopy = 10; + // let mut merkle_tree = IndexedMerkleTree::::new(height, canopy) + // .map_err(|_| IndexerError::InvalidResponseData)?; + // merkle_tree.merkle_tree.root_history_array_len = Some(STATE_MERKLE_TREE_ROOTS); + // let mut merkle_tree = Box::new(merkle_tree); + // merkle_tree.init()?; + // let mut indexed_array = Box::>::default(); + // indexed_array.init()?; // Test that the new config structure works correctly let config = StateUpdateConfig::default(); @@ -1216,6 +1283,8 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat base_indexed_seq, &mut v1_available_accounts_for_spending, &mut v2_available_accounts_for_spending, + &mut reference_indexed_array, + &mut reference_indexed_tree, ); println!("state_update {:?}", state_update); @@ -1268,8 +1337,8 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat &setup.db_conn, &metadata, &state_update, - &mut reference_indexed_tree, - &mut reference_indexed_array, + reference_indexed_tree.root(), + &reference_indexed_tree, ) .await .expect("Failed to verify indexed tree root"); From cb43f083529a70d616a7665fd6f9761ca591b2e7 Mon Sep 17 00:00:00 2001 From: ananas-block Date: Wed, 25 Jun 2025 20:39:16 +0100 Subject: [PATCH 18/21] possibly ided the issue --- .../persist/persisted_indexed_merkle_tree.rs | 8 +-- .../persist_state_update_test.rs | 54 +++++++++---------- 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/src/ingester/persist/persisted_indexed_merkle_tree.rs b/src/ingester/persist/persisted_indexed_merkle_tree.rs index f163074d..c65abf47 100644 --- a/src/ingester/persist/persisted_indexed_merkle_tree.rs +++ b/src/ingester/persist/persisted_indexed_merkle_tree.rs @@ -368,7 +368,7 @@ pub async fn persist_indexed_tree_updates( let tree_type = indexed_leaf_updates .values() .find(|update| update.tree == tree) - .map(|update| update.tree_type.clone()) + .map(|update| update.tree_type) .ok_or_else(|| { IngesterError::ParserError(format!( "No indexed tree leaf updates found for tree: {}. Cannot determine tree type.", @@ -406,7 +406,7 @@ pub async fn persist_indexed_tree_updates( (sdk_tree, zeroeth_leaf.leaf_index as u64), IndexedTreeLeafUpdate { tree, - tree_type: tree_type.clone(), + tree_type, hash: zeroeth_hash.0, leaf: RawIndexedElement { value: zeroeth_leaf.value.clone().try_into().map_err(|_e| { @@ -429,7 +429,7 @@ pub async fn persist_indexed_tree_updates( } // Check if top element (leaf_index 1) is missing and insert it if needed - ONLY for V1 trees - if matches!(tree_type, TreeType::AddressV1 | TreeType::StateV1) { + if matches!(tree_type, TreeType::AddressV1) { let top_update = indexed_leaf_updates.get(&(sdk_tree, 1)); if top_update.is_none() { let top_leaf = get_top_element(sdk_tree.to_bytes().to_vec()); @@ -441,7 +441,7 @@ pub async fn persist_indexed_tree_updates( (sdk_tree, top_leaf.leaf_index as u64), IndexedTreeLeafUpdate { tree, - tree_type: tree_type.clone(), + tree_type, hash: top_hash.0, leaf: RawIndexedElement { value: top_leaf.value.clone().try_into().map_err(|_e| { diff --git a/tests/integration_tests/persist_state_update_test.rs b/tests/integration_tests/persist_state_update_test.rs index 78f566bd..0a45f3fb 100644 --- a/tests/integration_tests/persist_state_update_test.rs +++ b/tests/integration_tests/persist_state_update_test.rs @@ -104,7 +104,7 @@ impl Default for StateUpdateConfig { account_transactions: CollectionConfig::new(0, 3, 0.0), transactions: CollectionConfig::new(0, 2, 0.0), leaf_nullifications: CollectionConfig::new(0, 3, 0.0), - indexed_merkle_tree_updates: CollectionConfig::new(0, 3, 1.0), + indexed_merkle_tree_updates: CollectionConfig::new(1, 1, 1.0), batch_nullify_context: CollectionConfig::new(0, 2, 0.0), batch_new_addresses: CollectionConfig::new(0, 3, 0.0), @@ -406,6 +406,7 @@ fn get_rnd_state_update( ) .unwrap(); let update = IndexedTreeLeafUpdate { + tree_type: TreeType::AddressV1, tree, leaf: RawIndexedElement { value: value_bytes, @@ -1028,7 +1029,7 @@ async fn assert_indexed_tree_root( // Get database root from state_trees table (indexed trees use same table structure) use photon_indexer::dao::generated::state_trees; - let ref_leaves = (0..4) + let ref_leaves = (0..(reference_tree.merkle_tree.get_next_index() - 1)) .map(|i| reference_tree.merkle_tree.get_leaf(i).unwrap()) .collect::>(); let mut leaf_nodes_from_db = state_trees::Entity::find() @@ -1043,18 +1044,20 @@ async fn assert_indexed_tree_root( let leaf_nodes = leaf_nodes_from_db .iter() .map(|x| { + println!("db leaf node {:?}", x); let mut hash_array = [0u8; 32]; hash_array.copy_from_slice(&x.hash); hash_array }) .collect::>(); - assert_eq!(leaf_nodes, ref_leaves); let root_nodes = state_trees::Entity::find() .filter(state_trees::Column::Tree.eq(tree_pubkey_bytes.clone())) .filter(state_trees::Column::Level.eq(26)) .all(db_conn) .await?; + println!("root_nodes {:?}", root_nodes); + assert_eq!(leaf_nodes, ref_leaves); println!("Database Indexed Tree Entries:"); for entry in &db_indexed_entries { @@ -1217,39 +1220,34 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat let mut reference_indexed_array = IndexedArray::::default(); reference_indexed_array.init().unwrap(); reference_indexed_tree.init().unwrap(); - // let height = 26; - // let canopy = 10; - // let mut merkle_tree = IndexedMerkleTree::::new(height, canopy) - // .map_err(|_| IndexerError::InvalidResponseData)?; - // merkle_tree.merkle_tree.root_history_array_len = Some(STATE_MERKLE_TREE_ROOTS); - // let mut merkle_tree = Box::new(merkle_tree); - // merkle_tree.init()?; - // let mut indexed_array = Box::>::default(); - // indexed_array.init()?; + let ref_leaves = (0..(reference_indexed_tree.merkle_tree.get_next_index() - 1)) + .map(|i| reference_indexed_tree.merkle_tree.get_leaf(i).unwrap()) + .collect::>(); + println!("Reference indexed leaves: {:?}", ref_leaves); // Test that the new config structure works correctly let config = StateUpdateConfig::default(); - // Verify config structure values - assert_eq!(config.in_accounts_v1.min_entries, 0); - assert_eq!(config.in_accounts_v1.max_entries, 3); - assert_eq!(config.in_accounts_v1.probability, 0.3); + // // Verify config structure values + // assert_eq!(config.in_accounts_v1.min_entries, 0); + // assert_eq!(config.in_accounts_v1.max_entries, 3); + // assert_eq!(config.in_accounts_v1.probability, 0.3); - assert_eq!(config.in_accounts_v2.min_entries, 0); - assert_eq!(config.in_accounts_v2.max_entries, 3); - assert_eq!(config.in_accounts_v2.probability, 0.3); + // assert_eq!(config.in_accounts_v2.min_entries, 0); + // assert_eq!(config.in_accounts_v2.max_entries, 3); + // assert_eq!(config.in_accounts_v2.probability, 0.3); - assert_eq!(config.out_accounts_v1.min_entries, 0); - assert_eq!(config.out_accounts_v1.max_entries, 5); - assert_eq!(config.out_accounts_v1.probability, 1.0); + // assert_eq!(config.out_accounts_v1.min_entries, 0); + // assert_eq!(config.out_accounts_v1.max_entries, 5); + // assert_eq!(config.out_accounts_v1.probability, 1.0); - assert_eq!(config.out_accounts_v2.min_entries, 0); - assert_eq!(config.out_accounts_v2.max_entries, 5); - assert_eq!(config.out_accounts_v2.probability, 1.0); + // assert_eq!(config.out_accounts_v2.min_entries, 0); + // assert_eq!(config.out_accounts_v2.max_entries, 5); + // assert_eq!(config.out_accounts_v2.probability, 1.0); - assert_eq!(config.transactions.min_entries, 0); - assert_eq!(config.transactions.max_entries, 2); - assert_eq!(config.transactions.probability, 0.0); + // assert_eq!(config.transactions.min_entries, 0); + // assert_eq!(config.transactions.max_entries, 2); + // assert_eq!(config.transactions.probability, 0.0); // Test that we can create a state update with incremental values let mut base_seq_v1 = 500; From d6941dd758e34d9b73045c7ffac972a4fa47c595 Mon Sep 17 00:00:00 2001 From: ananas-block Date: Wed, 25 Jun 2025 20:57:10 +0100 Subject: [PATCH 19/21] fixed the low element update in photon --- .../persist/persisted_indexed_merkle_tree.rs | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/src/ingester/persist/persisted_indexed_merkle_tree.rs b/src/ingester/persist/persisted_indexed_merkle_tree.rs index c65abf47..4f8a3c87 100644 --- a/src/ingester/persist/persisted_indexed_merkle_tree.rs +++ b/src/ingester/persist/persisted_indexed_merkle_tree.rs @@ -376,6 +376,134 @@ pub async fn persist_indexed_tree_updates( )) })?; + // Process low element updates - implement the missing indexed tree logic + // Build a local view of the tree to determine what low elements need updating + let tree_values: Vec> = indexed_leaf_updates + .iter() + .filter(|((tree_key, _), _)| *tree_key == sdk_tree) + .map(|(_, update)| update.leaf.value.to_vec()) + .collect(); + + if !tree_values.is_empty() { + // Query existing tree state + let mut indexed_tree = query_next_smallest_elements(txn, tree_values.clone(), sdk_tree.to_bytes().to_vec()).await?; + let mut elements_to_update: HashMap = HashMap::new(); + + // Add initialization elements if tree is empty + if indexed_tree.is_empty() { + let models = match &tree_type { + TreeType::AddressV1 | TreeType::StateV1 => { + vec![ + get_zeroeth_exclusion_range_v1(sdk_tree.to_bytes().to_vec()), + get_top_element(sdk_tree.to_bytes().to_vec()), + ] + } + _ => { + vec![get_zeroeth_exclusion_range(sdk_tree.to_bytes().to_vec())] + } + }; + for model in models { + elements_to_update.insert(model.leaf_index, model.clone()); + indexed_tree.insert(model.value.clone(), model); + } + } + + // Collect low element updates first to avoid borrow checker issues + let mut low_element_updates: HashMap<(Pubkey, u64), IndexedTreeLeafUpdate> = HashMap::new(); + + // Process each new element and update the tree structure + for ((tree_key, leaf_index), update) in &indexed_leaf_updates { + if *tree_key == sdk_tree && *leaf_index > 1 { // Skip initialization elements + let value = update.leaf.value.to_vec(); + + // Create the new indexed element + let mut new_element = indexed_trees::Model { + tree: sdk_tree.to_bytes().to_vec(), + leaf_index: *leaf_index as i64, + value: value.clone(), + next_index: 0, + next_value: vec![], + seq: Some(update.seq as i64), + }; + + // Find the low element (largest element smaller than the new value) + let low_element = indexed_tree + .range(..value.clone()) + .next_back() + .map(|(_, v)| v.clone()); + + if let Some(mut low_element) = low_element { + // Update the new element to point to what the low element was pointing to + new_element.next_index = low_element.next_index; + new_element.next_value = low_element.next_value.clone(); + + // Update the low element to point to the new element + low_element.next_index = *leaf_index as i64; + low_element.next_value = value.clone(); + + // Create the low element update + let low_element_hash = match &tree_type { + TreeType::AddressV1 | TreeType::StateV1 => { + compute_range_node_hash_v1(&low_element).map_err(|e| { + IngesterError::ParserError(format!( + "Failed to compute low element hash: {}", + e + )) + })?.0 + } + _ => { + compute_range_node_hash(&low_element).map_err(|e| { + IngesterError::ParserError(format!( + "Failed to compute low element hash: {}", + e + )) + })?.0 + } + }; + + let low_element_update = IndexedTreeLeafUpdate { + tree, + tree_type, + hash: low_element_hash, + leaf: RawIndexedElement { + value: low_element.value.clone().try_into().map_err(|_e| { + IngesterError::ParserError(format!( + "Failed to convert low element value to array {:?}", + low_element.value + )) + })?, + next_index: low_element.next_index as usize, + next_value: low_element.next_value.clone().try_into().map_err(|_e| { + IngesterError::ParserError( + "Failed to convert low element next value to array".to_string(), + ) + })?, + index: low_element.leaf_index as usize, + }, + seq: update.seq, // Use same sequence as the new element + }; + + // Collect the low element update + low_element_updates.insert( + (sdk_tree, low_element.leaf_index as u64), + low_element_update + ); + + // Update local tree state + elements_to_update.insert(low_element.leaf_index, low_element.clone()); + indexed_tree.insert(low_element.value.clone(), low_element); + } + + // Add the new element to local state + elements_to_update.insert(new_element.leaf_index, new_element.clone()); + indexed_tree.insert(value, new_element); + } + } + + // Apply the low element updates + indexed_leaf_updates.extend(low_element_updates); + } + // Check if zeroeth element (leaf_index 0) is missing and insert it if needed let zeroeth_update = indexed_leaf_updates.get(&(sdk_tree, 0)); if zeroeth_update.is_none() { From 7a5332ff14eaa921606cc80aa6e1f8353d0e9477 Mon Sep 17 00:00:00 2001 From: ananas-block Date: Wed, 25 Jun 2025 22:45:31 +0100 Subject: [PATCH 20/21] v1 address debugged --- src/ingester/error.rs | 2 + .../parser/merkle_tree_events_parser.rs | 2 +- .../persist/persisted_indexed_merkle_tree.rs | 407 ++++++++---------- .../persist_state_update_test.rs | 72 ++-- 4 files changed, 232 insertions(+), 251 deletions(-) diff --git a/src/ingester/error.rs b/src/ingester/error.rs index 12b87ef1..30b9ce44 100644 --- a/src/ingester/error.rs +++ b/src/ingester/error.rs @@ -14,6 +14,8 @@ pub enum IngesterError { EmptyBatchEvent, #[error("Invalid event.")] InvalidEvent, + #[error("Low element not found.")] + LowElementNotFound, } impl From for IngesterError { diff --git a/src/ingester/parser/merkle_tree_events_parser.rs b/src/ingester/parser/merkle_tree_events_parser.rs index 12a326b7..244a8749 100644 --- a/src/ingester/parser/merkle_tree_events_parser.rs +++ b/src/ingester/parser/merkle_tree_events_parser.rs @@ -130,7 +130,7 @@ fn parse_indexed_merkle_tree_update( { let indexed_tree_leaf_update = IndexedTreeLeafUpdate { tree: tree_pubkey, - tree_type: tree_type.clone(), + tree_type, hash: *hash, leaf: *leaf, seq, diff --git a/src/ingester/persist/persisted_indexed_merkle_tree.rs b/src/ingester/persist/persisted_indexed_merkle_tree.rs index 4f8a3c87..d80ac0d5 100644 --- a/src/ingester/persist/persisted_indexed_merkle_tree.rs +++ b/src/ingester/persist/persisted_indexed_merkle_tree.rs @@ -356,242 +356,207 @@ pub async fn get_exclusion_range_with_proof_v1( Ok((range_node.clone(), leaf_proof)) } -pub async fn persist_indexed_tree_updates( - txn: &DatabaseTransaction, - mut indexed_leaf_updates: HashMap<(Pubkey, u64), IndexedTreeLeafUpdate>, +/// Ensures the zeroeth element (leaf_index 0) exists if not already present +fn ensure_zeroeth_element_exists( + indexed_leaf_updates: &mut HashMap<(Pubkey, u64), IndexedTreeLeafUpdate>, + sdk_tree: Pubkey, + tree: Pubkey, + tree_type: TreeType, ) -> Result<(), IngesterError> { - let trees: HashSet = indexed_leaf_updates.keys().map(|x| x.0).collect(); - - for sdk_tree in trees { - let tree = Pubkey::new_from_array(sdk_tree.to_bytes()); - - let tree_type = indexed_leaf_updates - .values() - .find(|update| update.tree == tree) - .map(|update| update.tree_type) - .ok_or_else(|| { - IngesterError::ParserError(format!( - "No indexed tree leaf updates found for tree: {}. Cannot determine tree type.", - tree - )) - })?; - - // Process low element updates - implement the missing indexed tree logic - // Build a local view of the tree to determine what low elements need updating - let tree_values: Vec> = indexed_leaf_updates - .iter() - .filter(|((tree_key, _), _)| *tree_key == sdk_tree) - .map(|(_, update)| update.leaf.value.to_vec()) - .collect(); - - if !tree_values.is_empty() { - // Query existing tree state - let mut indexed_tree = query_next_smallest_elements(txn, tree_values.clone(), sdk_tree.to_bytes().to_vec()).await?; - let mut elements_to_update: HashMap = HashMap::new(); - - // Add initialization elements if tree is empty - if indexed_tree.is_empty() { - let models = match &tree_type { - TreeType::AddressV1 | TreeType::StateV1 => { - vec![ - get_zeroeth_exclusion_range_v1(sdk_tree.to_bytes().to_vec()), - get_top_element(sdk_tree.to_bytes().to_vec()), - ] - } - _ => { - vec![get_zeroeth_exclusion_range(sdk_tree.to_bytes().to_vec())] - } - }; - for model in models { - elements_to_update.insert(model.leaf_index, model.clone()); - indexed_tree.insert(model.value.clone(), model); - } + let zeroeth_update = indexed_leaf_updates.get(&(sdk_tree, 0)); + if zeroeth_update.is_none() { + let (zeroeth_leaf, zeroeth_hash) = match &tree_type { + TreeType::AddressV1 => { + let leaf = get_zeroeth_exclusion_range_v1(sdk_tree.to_bytes().to_vec()); + let hash = compute_range_node_hash_v1(&leaf).map_err(|e| { + IngesterError::ParserError(format!( + "Failed to compute zeroeth element hash: {}", + e + )) + })?; + (leaf, hash) } - - // Collect low element updates first to avoid borrow checker issues - let mut low_element_updates: HashMap<(Pubkey, u64), IndexedTreeLeafUpdate> = HashMap::new(); - - // Process each new element and update the tree structure - for ((tree_key, leaf_index), update) in &indexed_leaf_updates { - if *tree_key == sdk_tree && *leaf_index > 1 { // Skip initialization elements - let value = update.leaf.value.to_vec(); - - // Create the new indexed element - let mut new_element = indexed_trees::Model { - tree: sdk_tree.to_bytes().to_vec(), - leaf_index: *leaf_index as i64, - value: value.clone(), - next_index: 0, - next_value: vec![], - seq: Some(update.seq as i64), - }; - - // Find the low element (largest element smaller than the new value) - let low_element = indexed_tree - .range(..value.clone()) - .next_back() - .map(|(_, v)| v.clone()); - - if let Some(mut low_element) = low_element { - // Update the new element to point to what the low element was pointing to - new_element.next_index = low_element.next_index; - new_element.next_value = low_element.next_value.clone(); - - // Update the low element to point to the new element - low_element.next_index = *leaf_index as i64; - low_element.next_value = value.clone(); - - // Create the low element update - let low_element_hash = match &tree_type { - TreeType::AddressV1 | TreeType::StateV1 => { - compute_range_node_hash_v1(&low_element).map_err(|e| { - IngesterError::ParserError(format!( - "Failed to compute low element hash: {}", - e - )) - })?.0 - } - _ => { - compute_range_node_hash(&low_element).map_err(|e| { - IngesterError::ParserError(format!( - "Failed to compute low element hash: {}", - e - )) - })?.0 - } - }; - - let low_element_update = IndexedTreeLeafUpdate { - tree, - tree_type, - hash: low_element_hash, - leaf: RawIndexedElement { - value: low_element.value.clone().try_into().map_err(|_e| { - IngesterError::ParserError(format!( - "Failed to convert low element value to array {:?}", - low_element.value - )) - })?, - next_index: low_element.next_index as usize, - next_value: low_element.next_value.clone().try_into().map_err(|_e| { - IngesterError::ParserError( - "Failed to convert low element next value to array".to_string(), - ) - })?, - index: low_element.leaf_index as usize, - }, - seq: update.seq, // Use same sequence as the new element - }; - - // Collect the low element update - low_element_updates.insert( - (sdk_tree, low_element.leaf_index as u64), - low_element_update - ); - - // Update local tree state - elements_to_update.insert(low_element.leaf_index, low_element.clone()); - indexed_tree.insert(low_element.value.clone(), low_element); - } - - // Add the new element to local state - elements_to_update.insert(new_element.leaf_index, new_element.clone()); - indexed_tree.insert(value, new_element); - } + _ => { + let leaf = get_zeroeth_exclusion_range(sdk_tree.to_bytes().to_vec()); + let hash = compute_range_node_hash(&leaf).map_err(|e| { + IngesterError::ParserError(format!( + "Failed to compute zeroeth element hash: {}", + e + )) + })?; + (leaf, hash) } - - // Apply the low element updates - indexed_leaf_updates.extend(low_element_updates); - } + }; - // Check if zeroeth element (leaf_index 0) is missing and insert it if needed - let zeroeth_update = indexed_leaf_updates.get(&(sdk_tree, 0)); - if zeroeth_update.is_none() { - let (zeroeth_leaf, zeroeth_hash) = match &tree_type { - TreeType::AddressV1 | TreeType::StateV1 => { - let leaf = get_zeroeth_exclusion_range_v1(sdk_tree.to_bytes().to_vec()); - let hash = compute_range_node_hash_v1(&leaf).map_err(|e| { + indexed_leaf_updates.insert( + (sdk_tree, zeroeth_leaf.leaf_index as u64), + IndexedTreeLeafUpdate { + tree, + tree_type, + hash: zeroeth_hash.0, + leaf: RawIndexedElement { + value: zeroeth_leaf.value.clone().try_into().map_err(|_e| { IngesterError::ParserError(format!( - "Failed to compute zeroeth element hash: {}", - e + "Failed to convert zeroeth element value to array {:?}", + zeroeth_leaf.value )) - })?; - (leaf, hash) - } - _ => { - let leaf = get_zeroeth_exclusion_range(sdk_tree.to_bytes().to_vec()); - let hash = compute_range_node_hash(&leaf).map_err(|e| { - IngesterError::ParserError(format!( - "Failed to compute zeroeth element hash: {}", - e - )) - })?; - (leaf, hash) - } - }; + })?, + next_index: zeroeth_leaf.next_index as usize, + next_value: zeroeth_leaf.next_value.try_into().map_err(|_e| { + IngesterError::ParserError( + "Failed to convert zeroeth element next value to array".to_string(), + ) + })?, + index: zeroeth_leaf.leaf_index as usize, + }, + seq: 0, + }, + ); + } + Ok(()) +} + +/// Ensures the top element (leaf_index 1) exists for V1 trees if not already present +fn ensure_top_element_exists( + indexed_leaf_updates: &mut HashMap<(Pubkey, u64), IndexedTreeLeafUpdate>, + sdk_tree: Pubkey, + tree: Pubkey, + tree_type: TreeType, +) -> Result<(), IngesterError> { + // Check if top element (leaf_index 1) is missing and insert it if needed - ONLY for V1 trees + if matches!(tree_type, TreeType::AddressV1) { + let top_update = indexed_leaf_updates.get(&(sdk_tree, 1)); + if top_update.is_none() { + let top_leaf = get_top_element(sdk_tree.to_bytes().to_vec()); + let top_hash = compute_range_node_hash_v1(&top_leaf).map_err(|e| { + IngesterError::ParserError(format!("Failed to compute top element hash: {}", e)) + })?; indexed_leaf_updates.insert( - (sdk_tree, zeroeth_leaf.leaf_index as u64), + (sdk_tree, top_leaf.leaf_index as u64), IndexedTreeLeafUpdate { tree, tree_type, - hash: zeroeth_hash.0, + hash: top_hash.0, leaf: RawIndexedElement { - value: zeroeth_leaf.value.clone().try_into().map_err(|_e| { + value: top_leaf.value.clone().try_into().map_err(|_e| { IngesterError::ParserError(format!( - "Failed to convert zeroeth element value to array {:?}", - zeroeth_leaf.value + "Failed to convert top element value to array {:?}", + top_leaf.value )) })?, - next_index: zeroeth_leaf.next_index as usize, - next_value: zeroeth_leaf.next_value.try_into().map_err(|_e| { + next_index: top_leaf.next_index as usize, + next_value: top_leaf.next_value.try_into().map_err(|_e| { IngesterError::ParserError( - "Failed to convert zeroeth element next value to array".to_string(), + "Failed to convert top element next value to array".to_string(), ) })?, - index: zeroeth_leaf.leaf_index as usize, + index: top_leaf.leaf_index as usize, }, - seq: 0, + seq: 1, }, ); } + } + Ok(()) +} - // Check if top element (leaf_index 1) is missing and insert it if needed - ONLY for V1 trees - if matches!(tree_type, TreeType::AddressV1) { - let top_update = indexed_leaf_updates.get(&(sdk_tree, 1)); - if top_update.is_none() { - let top_leaf = get_top_element(sdk_tree.to_bytes().to_vec()); - let top_hash = compute_range_node_hash_v1(&top_leaf).map_err(|e| { - IngesterError::ParserError(format!("Failed to compute top element hash: {}", e)) - })?; +/// Creates a low element update from the given parameters +fn create_low_element_update( + low_element: &indexed_trees::Model, + tree: Pubkey, + tree_type: TreeType, + seq: u64, +) -> Result { + let low_element_hash = match &tree_type { + TreeType::AddressV1 => Ok(compute_range_node_hash_v1(low_element) + .map_err(|e| { + IngesterError::ParserError(format!("Failed to compute low element hash: {}", e)) + })? + .0), + TreeType::AddressV2 => Ok(compute_range_node_hash(low_element) + .map_err(|e| { + IngesterError::ParserError(format!("Failed to compute low element hash: {}", e)) + })? + .0), + _ => Err(IngesterError::ParserError(format!( + "Invalid tree type: {}", + tree_type + ))), + }?; + + let low_element_update = IndexedTreeLeafUpdate { + tree, + tree_type, + hash: low_element_hash, + leaf: RawIndexedElement { + value: low_element.value.clone().try_into().map_err(|_e| { + IngesterError::ParserError(format!( + "Failed to convert low element value to array {:?}", + low_element.value + )) + })?, + next_index: low_element.next_index as usize, + next_value: low_element.next_value.clone().try_into().map_err(|_e| { + IngesterError::ParserError( + "Failed to convert low element next value to array".to_string(), + ) + })?, + index: low_element.leaf_index as usize, + }, + seq, + }; - indexed_leaf_updates.insert( - (sdk_tree, top_leaf.leaf_index as u64), - IndexedTreeLeafUpdate { - tree, - tree_type, - hash: top_hash.0, - leaf: RawIndexedElement { - value: top_leaf.value.clone().try_into().map_err(|_e| { - IngesterError::ParserError(format!( - "Failed to convert top element value to array {:?}", - top_leaf.value - )) - })?, - next_index: top_leaf.next_index as usize, - next_value: top_leaf.next_value.try_into().map_err(|_e| { - IngesterError::ParserError( - "Failed to convert top element next value to array".to_string(), - ) - })?, - index: top_leaf.leaf_index as usize, - }, - seq: 1, - }, - ); - } - } + Ok(low_element_update) +} + +/// Persists indexed Merkle tree updates to the database, maintaining the linked structure +/// required for indexed trees where each element points to the next element in sorted order. +/// +/// This function implements indexed Merkle tree operations including both new element +/// appends and the corresponding low element updates that maintain tree integrity. +/// +/// ## Steps performed: +/// 1. **Tree Processing**: Iterate through each unique tree in the updates +/// 2. **Tree Type Detection**: Determine if tree is V1 (AddressV1/StateV1) or V2 for proper hash computation +/// 3. **Low Element Updates**: +/// - Query existing tree state from database to build local view +/// - For empty trees, initialize with zeroeth and top elements as needed +/// - For each new element being appended: +/// - Find the "low element" (largest existing element smaller than new value) +/// - Update the low element to point to the new element (update its next_index/next_value) +/// - Configure the new element to point to what the low element was pointing to +/// - Recompute hashes for the updated low element +/// - Add low element update to the batch +/// 4. **Initialization Elements**: Ensure required initialization elements exist: +/// - Zeroeth element (leaf_index 0): Points to first real element or top element +/// - Top element (leaf_index 1): Only for V1 trees, represents the maximum value +/// 5. **Database Persistence**: +/// - Batch updates into chunks to avoid SQL parameter limits +/// - Use upsert logic with sequence number checks to handle conflicts +/// - Insert/update records in indexed_trees table +/// 6. **State Tree Integration**: Create corresponding leaf nodes for the Merkle tree structure +/// +pub async fn persist_indexed_tree_updates( + txn: &DatabaseTransaction, + mut indexed_leaf_updates: HashMap<(Pubkey, u64), IndexedTreeLeafUpdate>, +) -> Result<(), IngesterError> { + // Step 1: Tree Processing - Collect unique trees with their types + let trees: HashMap = indexed_leaf_updates + .values() + .map(|update| (update.tree, update.tree_type)) + .collect(); + + for (tree, tree_type) in trees { + let sdk_tree = Pubkey::new_from_array(tree.to_bytes()); + + // Step 4: Initialization Elements - Ensure required initialization elements exist + ensure_zeroeth_element_exists(&mut indexed_leaf_updates, sdk_tree, tree, tree_type)?; + + ensure_top_element_exists(&mut indexed_leaf_updates, sdk_tree, tree, tree_type)?; } + + // Step 5: Database Persistence - Batch updates and insert/update records let chunks = indexed_leaf_updates .values() .chunks(MAX_SQL_INSERTS) @@ -631,6 +596,7 @@ pub async fn persist_indexed_tree_updates( IngesterError::DatabaseError(format!("Failed to insert indexed tree elements: {}", e)) })?; + // Step 6: State Tree Integration - Create corresponding leaf nodes for the Merkle tree structure let state_tree_leaf_nodes = chunk .iter() .map(|x| { @@ -687,10 +653,11 @@ pub async fn multi_append( None => 0, }; - let mut indexed_tree = query_next_smallest_elements(txn, values.clone(), tree.clone()).await?; + let mut local_tmp_indexed_tree = + query_next_smallest_elements(txn, values.clone(), tree.clone()).await?; let mut elements_to_update: HashMap = HashMap::new(); - if indexed_tree.is_empty() { + if local_tmp_indexed_tree.is_empty() { let models = if tree_height == TREE_HEIGHT_V1 + 1 { vec![ get_zeroeth_exclusion_range_v1(tree.clone()), @@ -701,7 +668,7 @@ pub async fn multi_append( }; for model in models { elements_to_update.insert(model.leaf_index, model.clone()); - indexed_tree.insert(model.value.clone(), model); + local_tmp_indexed_tree.insert(model.value.clone(), model); } } @@ -716,7 +683,7 @@ pub async fn multi_append( seq: seq.map(|s| s as i64), }; - let next_largest = indexed_tree + let next_largest = local_tmp_indexed_tree .range(..value.clone()) // This ranges from the start up to, but not including, `key` .next_back() // Gets the last element in the range, which is the largest key less than `key` .map(|(_, v)| v.clone()); @@ -729,10 +696,10 @@ pub async fn multi_append( next_largest.next_value = value.clone(); elements_to_update.insert(next_largest.leaf_index, next_largest.clone()); - indexed_tree.insert(next_largest.value.clone(), next_largest); + local_tmp_indexed_tree.insert(next_largest.value.clone(), next_largest); } elements_to_update.insert(current_index, indexed_element.clone()); - indexed_tree.insert(value, indexed_element); + local_tmp_indexed_tree.insert(value, indexed_element); } let active_elements: Vec = elements_to_update @@ -852,7 +819,7 @@ where _ => unimplemented!(), }; - let mut indexed_tree: BTreeMap, indexed_trees::Model> = BTreeMap::new(); + let mut local_tmp_indexed_tree: BTreeMap, indexed_trees::Model> = BTreeMap::new(); for row in response { let model = indexed_trees::Model { tree: row.try_get("", "tree")?, @@ -862,9 +829,9 @@ where next_value: row.try_get("", "next_value")?, seq: row.try_get("", "seq")?, }; - indexed_tree.insert(model.value.clone(), model); + local_tmp_indexed_tree.insert(model.value.clone(), model); } - Ok(indexed_tree) + Ok(local_tmp_indexed_tree) } pub fn format_bytes(bytes: Vec, database_backend: DatabaseBackend) -> String { diff --git a/tests/integration_tests/persist_state_update_test.rs b/tests/integration_tests/persist_state_update_test.rs index 0a45f3fb..1fa5f7a3 100644 --- a/tests/integration_tests/persist_state_update_test.rs +++ b/tests/integration_tests/persist_state_update_test.rs @@ -104,7 +104,7 @@ impl Default for StateUpdateConfig { account_transactions: CollectionConfig::new(0, 3, 0.0), transactions: CollectionConfig::new(0, 2, 0.0), leaf_nullifications: CollectionConfig::new(0, 3, 0.0), - indexed_merkle_tree_updates: CollectionConfig::new(1, 1, 1.0), + indexed_merkle_tree_updates: CollectionConfig::new(0, 3, 1.0), batch_nullify_context: CollectionConfig::new(0, 2, 0.0), batch_new_addresses: CollectionConfig::new(0, 3, 0.0), @@ -363,7 +363,8 @@ fn get_rnd_state_update( config.indexed_merkle_tree_updates.min_entries ..=config.indexed_merkle_tree_updates.max_entries, ); - metadata.indexed_merkle_tree_updates_count = count; + let mut base_indexed_seq = base_indexed_seq; + // Two tree operations per indexed append for i in 0..count { let tree = solana_pubkey::pubkey!("amt1Ayt45jfbdw5YSo7iz6WZxUmnZsQTYXy82hVwyC2"); @@ -372,32 +373,20 @@ fn get_rnd_state_update( value_bytes[0] = 0; let value = BigUint::from_bytes_be(&value_bytes); - // // Find the low element for this value using the indexed array - // let (old_low_element, old_low_element_next_value) = indexed_array - // .find_low_element_for_nonexistent(&value) - // .unwrap(); - - // // Create the nullifier bundle which contains all the proper elements - // let nullifier_bundle = indexed_array - // .new_element_with_low_element_index(old_low_element.index, &value) - // .unwrap(); - - // // Use the bundle data to create the update - // let leaf_index = nullifier_bundle.new_element.index; - // let next_index = nullifier_bundle.new_element.next_index; - // let next_value_bytes = - // bigint_to_be_bytes_array::<32>(&nullifier_bundle.new_element_next_value).unwrap(); - - // // Compute the proper hash for the indexed element - // let hash = nullifier_bundle - // .new_element - // .hash::(&nullifier_bundle.new_element_next_value) - // .unwrap(); let nullifier_bundle = indexed_array.append(&value).unwrap(); - let hash = nullifier_bundle + + // Compute the hash for the new element + let new_element_hash = nullifier_bundle .new_element .hash::(&nullifier_bundle.new_element_next_value) .unwrap(); + + // Compute the hash for the low element (after its next_value was updated) + let low_element_hash = nullifier_bundle + .new_low_element + .hash::(&nullifier_bundle.new_element.value) + .unwrap(); + reference_indexed_tree .update( &nullifier_bundle.new_low_element, @@ -405,7 +394,26 @@ fn get_rnd_state_update( &nullifier_bundle.new_element_next_value, ) .unwrap(); - let update = IndexedTreeLeafUpdate { + let low_element_update = IndexedTreeLeafUpdate { + tree_type: TreeType::AddressV1, + tree, + leaf: RawIndexedElement { + value: bigint_to_be_bytes_array::<32>(&nullifier_bundle.new_low_element.value) + .unwrap(), + next_index: nullifier_bundle.new_low_element.next_index, + next_value: bigint_to_be_bytes_array::<32>(&nullifier_bundle.new_element.value) + .unwrap(), + index: nullifier_bundle.new_low_element.index, + }, + hash: low_element_hash, + seq: base_indexed_seq, + }; + base_indexed_seq += 1; + state_update.indexed_merkle_tree_updates.insert( + (tree, nullifier_bundle.new_low_element.index as u64), + low_element_update, + ); + let new_element_update = IndexedTreeLeafUpdate { tree_type: TreeType::AddressV1, tree, leaf: RawIndexedElement { @@ -417,13 +425,16 @@ fn get_rnd_state_update( .unwrap(), index: nullifier_bundle.new_element.index, }, - hash, - seq: base_indexed_seq + i as u64, + hash: new_element_hash, + seq: base_indexed_seq, }; - state_update - .indexed_merkle_tree_updates - .insert((tree, nullifier_bundle.new_element.index as u64), update); + base_indexed_seq += 1; + state_update.indexed_merkle_tree_updates.insert( + (tree, nullifier_bundle.new_element.index as u64), + new_element_update, + ); } + metadata.indexed_merkle_tree_updates_count = state_update.indexed_merkle_tree_updates.len(); } // Generate batch_nullify_context (Vec) for actual input accounts @@ -986,6 +997,7 @@ async fn assert_indexed_tree_root( use photon_indexer::dao::generated::indexed_trees; use sea_orm::ColumnTrait; + // Assert doesn't work because // Validate metadata consistency assert_eq!( state_update.indexed_merkle_tree_updates.len(), From 67e7c34541006997acb629e9a82d4ba6dc6408d9 Mon Sep 17 00:00:00 2001 From: ananas-block Date: Wed, 25 Jun 2025 23:18:46 +0100 Subject: [PATCH 21/21] add v2 address queue inserts --- .../persist_state_update_test.rs | 90 +++++++++++++++++-- 1 file changed, 83 insertions(+), 7 deletions(-) diff --git a/tests/integration_tests/persist_state_update_test.rs b/tests/integration_tests/persist_state_update_test.rs index 1fa5f7a3..190970b2 100644 --- a/tests/integration_tests/persist_state_update_test.rs +++ b/tests/integration_tests/persist_state_update_test.rs @@ -106,7 +106,7 @@ impl Default for StateUpdateConfig { leaf_nullifications: CollectionConfig::new(0, 3, 0.0), indexed_merkle_tree_updates: CollectionConfig::new(0, 3, 1.0), batch_nullify_context: CollectionConfig::new(0, 2, 0.0), - batch_new_addresses: CollectionConfig::new(0, 3, 0.0), + batch_new_addresses: CollectionConfig::new(1, 3, 1.0), lamports_min: 1000, lamports_max: 1_000_000, @@ -249,7 +249,7 @@ fn get_rnd_state_update( let count = rng.gen_range(config.out_accounts_v2.min_entries..=config.out_accounts_v2.max_entries); - metadata.out_accounts_v2_count = count as usize; + metadata.out_accounts_v2_count = count; for i in 0..count { let account = AccountWithContext { account: Account { @@ -453,16 +453,23 @@ fn get_rnd_state_update( metadata.batch_nullify_context_count += 1; } - // Generate batch_new_addresses (Vec) + // Generate batch_new_addresses (Vec) - V2 addresses for address queue if rng.gen_bool(config.batch_new_addresses.probability) { + // Use V2 tree for new addresses (they go into address queue) + let v2_tree_info = TreeInfo::get(V2_TEST_TREE_PUBKEY_STR) + .expect("V2 test tree should exist in QUEUE_TREE_MAPPING"); + let v2_tree_pubkey = v2_tree_info.tree; + let count = rng.gen_range( config.batch_new_addresses.min_entries..=config.batch_new_addresses.max_entries, ); + metadata.batch_new_addresses_count = count; + for i in 0..count { state_update.batch_new_addresses.push(AddressQueueUpdate { - tree: SerializablePubkey::from(test_tree_pubkey), + tree: SerializablePubkey::from(v2_tree_pubkey), address: rng.gen::<[u8; 32]>(), - queue_index: i as u64, + queue_index: nullifier_queue_index + i as u64, // Continue from where nullifier queue left off }); } } @@ -549,7 +556,7 @@ fn update_test_state_after_iteration( *base_seq_v1 += v1_output_count; *base_leaf_index_v1 += v1_output_count; *base_leaf_index_v2 += v2_output_count; - *base_nullifier_queue_index += v2_input_count; // Only v2 input accounts get nullifier queue positions + *base_nullifier_queue_index += v2_input_count + metadata.batch_new_addresses_count as u64; // V2 input accounts and new addresses share queue space *base_indexed_seq += indexed_updates_count; // Track indexed tree sequence println!( @@ -808,6 +815,70 @@ async fn assert_input_accounts_persisted( Ok(()) } +/// Assert that all batch_new_addresses from the state update were inserted correctly into the address queue +async fn assert_batch_new_addresses_persisted( + db_conn: &DatabaseConnection, + metadata: &StateUpdateMetadata, + state_update: &StateUpdate, +) -> Result<(), Box> { + use photon_indexer::dao::generated::address_queues; + use sea_orm::ColumnTrait; + + // Validate metadata matches actual state update + assert_eq!( + state_update.batch_new_addresses.len(), + metadata.batch_new_addresses_count, + "Metadata batch_new_addresses count ({}) doesn't match actual batch_new_addresses ({})", + metadata.batch_new_addresses_count, + state_update.batch_new_addresses.len() + ); + + if state_update.batch_new_addresses.is_empty() { + println!("✅ No batch_new_addresses - skipping address queue verification"); + return Ok(()); + } + + // Create expected models from state update + let expected_models: Vec = state_update + .batch_new_addresses + .iter() + .map(|address_update| address_queues::Model { + address: address_update.address.to_vec(), + tree: address_update.tree.0.to_bytes().to_vec(), + queue_index: address_update.queue_index as i64, + }) + .collect(); + + // Get all addresses for the query + let expected_addresses: Vec> = expected_models + .iter() + .map(|model| model.address.clone()) + .collect(); + + // Query database for addresses with matching addresses + let mut db_addresses = address_queues::Entity::find() + .filter(address_queues::Column::Address.is_in(expected_addresses)) + .all(db_conn) + .await?; + + // Sort both vectors by address for consistent comparison + let mut expected_models_sorted = expected_models; + expected_models_sorted.sort_by(|a, b| a.address.cmp(&b.address)); + db_addresses.sort_by(|a, b| a.address.cmp(&b.address)); + + // Single assert comparing the entire vectors + assert_eq!( + db_addresses, expected_models_sorted, + "Database addresses do not match expected addresses" + ); + + println!( + "✅ Successfully verified {} batch_new_addresses were persisted correctly in address queue", + db_addresses.len() + ); + Ok(()) +} + /// Assert that state tree root matches reference implementation after appending new hashes async fn assert_state_tree_root( db_conn: &DatabaseConnection, @@ -1221,7 +1292,7 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat // Set up deterministic randomness following the light-protocol pattern let mut thread_rng = ThreadRng::default(); let random_seed = thread_rng.next_u64(); - let seed: u64 = 5331180609400104349; //random_seed; + let seed: u64 = random_seed; println!("\n\nconfig structure test seed {}\n\n", seed); let mut rng = StdRng::seed_from_u64(seed); @@ -1330,6 +1401,11 @@ async fn test_output_accounts(#[values(DatabaseBackend::Sqlite)] db_backend: Dat .await .expect("Failed to verify input accounts persistence"); + // 5.5. Assert that all batch_new_addresses were persisted correctly in address queue + assert_batch_new_addresses_persisted(&setup.db_conn, &metadata, &state_update) + .await + .expect("Failed to verify batch_new_addresses persistence"); + // 6. Assert that state tree root matches reference tree root // - updates reference tree assert_state_tree_root(