diff --git a/Cargo.lock b/Cargo.lock index 0cb0a50b87..37f9eebde6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3633,6 +3633,7 @@ name = "miden-node-store" version = "0.16.0-alpha.3" dependencies = [ "anyhow", + "arc-swap", "assert_matches", "build-rs", "criterion", diff --git a/Cargo.toml b/Cargo.toml index ca34e80771..4939e9e1df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,6 +66,7 @@ miden-crypto = { version = "0.28" } # External dependencies anyhow = { version = "1.0" } +arc-swap = { version = "1.7" } assert_matches = { version = "1.5" } axum = { version = "0.8" } backon = { version = "1.6" } diff --git a/bin/node/src/commands/modes.rs b/bin/node/src/commands/modes.rs index ac6cbab936..56fbfb16e0 100644 --- a/bin/node/src/commands/modes.rs +++ b/bin/node/src/commands/modes.rs @@ -14,7 +14,7 @@ use miden_node_proto::clients::{ WantsConnection, }; use miden_node_rpc::{PreAuthSubmission, Rpc, RpcMode, SequencerInternal, ValidatorClients}; -use miden_node_store::State; +use miden_node_store::{BlockWriter, ProofWriter, State, WriterTask}; use miden_node_utils::clap::{GrpcOptionsInternal, duration_to_human_readable_string}; use miden_node_utils::formatting::format_endpoint; use miden_node_utils::shutdown::CancellationToken; @@ -67,11 +67,14 @@ impl SequencerCommand { remote_prover_monitor(self.block_producer.batch.prover_url.as_ref())?; let block_prover_monitor = remote_prover_monitor(self.block_producer.block_prover.url.as_ref())?; - let state = load_state(&runtime).await?; + let (state, block_writer, proof_writer, writer_task) = + load_state(&runtime, shutdown.clone()).await?; let _disk_monitor = state.spawn_disk_monitor(shutdown.clone()); let sequencer = Sequencer { - store: Arc::clone(&state), + state: Arc::clone(&state), + block_writer, + proof_writer, validator_urls: self.external_services.validator_urls.clone(), validator_timeout: self.external_services.validator_timeout, batch_prover_url: self.block_producer.batch.prover_url, @@ -85,14 +88,14 @@ impl SequencerCommand { batch_workers: self.block_producer.batch.workers, } .spawn(shutdown.clone()) - .await .context("failed to spawn sequencer")?; let block_producer = sequencer.api(); let rpc = Rpc { listener: bind_rpc(runtime.rpc_listen).await?, - store: state, + state, mode: RpcMode::sequencer(block_producer.clone(), validator_clients), + sync_writers: None, ntx_builder: Some(ntx_builder_client), grpc_options: runtime.external_grpc_options, network_tx_auth, @@ -100,6 +103,7 @@ impl SequencerCommand { let mut tasks = Tasks::new(); tasks.spawn("sequencer", sequencer.wait()); tasks.spawn("RPC server", rpc.serve(shutdown.clone())); + tasks.spawn("store block writer", join_store_writer(writer_task)); for (index, validator_monitor) in validator_monitors.into_iter().enumerate() { tasks.spawn_infallible( format!("validator {index} connection monitor"), @@ -288,19 +292,22 @@ impl FullNodeCommand { sequencer_monitor, } = self.pre_auth_components()?; let network_tx_auth = self.runtime.rpc.network_tx_auth()?; - let state = load_state(&runtime).await?; + let (state, block_writer, proof_writer, writer_task) = + load_state(&runtime, shutdown.clone()).await?; let _disk_monitor = state.spawn_disk_monitor(shutdown.clone()); let rpc = Rpc { listener: bind_rpc(runtime.rpc_listen).await?, - store: state, + state, mode: RpcMode::full_node(source_rpc, self.sync.readiness_threshold, pre_auth), + sync_writers: Some((block_writer, proof_writer)), ntx_builder: None, grpc_options: runtime.external_grpc_options, network_tx_auth, }; let mut tasks = Tasks::new(); tasks.spawn("RPC server", rpc.serve(shutdown.clone())); + tasks.spawn("store block writer", join_store_writer(writer_task)); for (index, validator_monitor) in validator_monitors.into_iter().enumerate() { tasks.spawn_infallible( format!("validator {index} connection monitor"), @@ -399,8 +406,11 @@ impl SyncOptions { } } -async fn load_state(runtime: &RuntimeConfig) -> anyhow::Result> { - let state = State::load_with_database_options( +async fn load_state( + runtime: &RuntimeConfig, + shutdown: CancellationToken, +) -> anyhow::Result<(Arc, BlockWriter, ProofWriter, WriterTask)> { + let loaded = State::load_with_database_options( &runtime.data_directory, runtime.storage_options.clone(), runtime.database_options, @@ -408,7 +418,16 @@ async fn load_state(runtime: &RuntimeConfig) -> anyhow::Result> { .await .context("failed to load state")?; - Ok(Arc::new(state)) + Ok(loaded.start(shutdown)) +} + +/// Supervises the store's write worker task. +/// +/// On shutdown the task-drain loop waits for the writer to finish any in-flight block write and +/// close its storage; an early exit or panic surfaces through the task set like any other task +/// failure. +async fn join_store_writer(writer_task: WriterTask) -> anyhow::Result<()> { + writer_task.await.map_err(anyhow::Error::from) } async fn bind_rpc(listen: SocketAddr) -> anyhow::Result { diff --git a/bin/node/src/commands/recover.rs b/bin/node/src/commands/recover.rs index 343850f998..a4abe94fc5 100644 --- a/bin/node/src/commands/recover.rs +++ b/bin/node/src/commands/recover.rs @@ -5,8 +5,8 @@ use std::time::Duration; use anyhow::Context; use miden_node_proto::clients::{Builder, ValidatorClient}; use miden_node_proto::generated::validator::BlockSubscriptionRequest; -use miden_node_store::State; -use miden_node_store::state::Finality; +use miden_node_store::{BlockWriter, State, WriterTask}; +use miden_node_utils::shutdown::CancellationToken; use miden_protocol::block::{BlockNumber, SignedBlock}; use miden_protocol::utils::serde::Deserializable; use tokio_stream::StreamExt; @@ -36,13 +36,18 @@ pub struct RecoverCommand { impl RecoverCommand { pub async fn handle(self) -> anyhow::Result<()> { - let state = self.load_state().await?; + let (state, mut block_writer, writer_task) = self.load_state().await?; let validator = self.validator_client()?; - recover_from_validator(&state, validator).await + let result = recover_from_validator(&state, &mut block_writer, validator).await; + // Wait for the writer to drain and release the backing storage before the process exits. + block_writer.stop(writer_task).await; + result } - async fn load_state(&self) -> anyhow::Result> { - let state = State::load_with_database_options( + async fn load_state(&self) -> anyhow::Result<(Arc, BlockWriter, WriterTask)> { + // Recovery is not wired into the node's shutdown token; the writer exits once the + // `BlockWriter` (holding the only write handle) is dropped after recovery completes. + let loaded = State::load_with_database_options( &self.data_directory, self.store.storage.clone().into(), self.store.sqlite.database_options(), @@ -50,7 +55,9 @@ impl RecoverCommand { .await .context("failed to load state")?; - Ok(Arc::new(state)) + let (state, block_writer, _proof_writer, writer_task) = + loaded.start(CancellationToken::new()); + Ok((state, block_writer, writer_task)) } fn validator_client(&self) -> anyhow::Result { @@ -66,7 +73,8 @@ impl RecoverCommand { /// Streams blocks from the validator into the local store until the chain tip is reached. async fn recover_from_validator( - state: &Arc, + state: &State, + block_writer: &mut BlockWriter, mut validator: ValidatorClient, ) -> anyhow::Result<()> { // Capture the validator's chain tip as the recovery target. The validator's block stream @@ -81,7 +89,7 @@ async fn recover_from_validator( .chain_tip, ); - let local_tip = state.chain_tip(Finality::Committed).await; + let local_tip = state.committed_tip(); if local_tip >= validator_tip { info!( target: LOG_TARGET, @@ -111,7 +119,10 @@ async fn recover_from_validator( let block = SignedBlock::read_from_bytes(&event.block) .context("failed to deserialize block from validator")?; let block_num = block.header().block_num(); - state.apply_block(block).await.context("failed to apply recovered block")?; + block_writer + .apply_block(block) + .await + .context("failed to apply recovered block")?; info!(target: LOG_TARGET, block_number = %block_num.as_u32(), "Applied recovered block"); // Stop once we reach the tip captured at the start of recovery. @@ -121,7 +132,7 @@ async fn recover_from_validator( } // The stream can end before reaching the tip if the validator restarts or drops the connection. - let final_tip = state.chain_tip(Finality::Committed).await; + let final_tip = state.committed_tip(); anyhow::ensure!( final_tip >= validator_tip, "validator block stream ended at block {} before reaching the chain tip {}", diff --git a/bin/stress-test/src/seeding/mod.rs b/bin/stress-test/src/seeding/mod.rs index eeb21fae5d..dadb177a9f 100644 --- a/bin/stress-test/src/seeding/mod.rs +++ b/bin/stress-test/src/seeding/mod.rs @@ -6,8 +6,9 @@ use std::time::Instant; use metrics::SeedingMetrics; use miden_node_proto::domain::batch::BatchInputs; -use miden_node_store::{DataDirectory, GenesisState, State}; +use miden_node_store::{BlockWriter, DataDirectory, GenesisState, State, WriterTask}; use miden_node_utils::clap::StorageOptions; +use miden_node_utils::shutdown::CancellationToken; use miden_protocol::account::auth::AuthScheme; use miden_protocol::account::{ Account, @@ -235,7 +236,7 @@ pub async fn seed_store_with_readers( let genesis_header = genesis_block.inner().header().clone(); State::bootstrap(genesis_block, &data_directory).expect("store should bootstrap"); - let store_state = load_state(data_directory.clone()).await; + let (state, mut block_writer, writer_task) = load_state(data_directory.clone()).await; // Recreate the deterministic genesis benchmark accounts after bootstrapping instead of keeping // another copy of their potentially very large maps alive while the genesis block is built. @@ -259,7 +260,7 @@ pub async fn seed_store_with_readers( let stop_readers = Arc::new(AtomicBool::new(false)); let reader_tasks: Vec<_> = (0..readers) .map(|_| { - let state = Arc::clone(&store_state); + let state = Arc::clone(&state); let stop = Arc::clone(&stop_readers); tokio::spawn(async move { read_latest_header_until(&state, &stop).await }) }) @@ -270,6 +271,7 @@ pub async fn seed_store_with_readers( let data_directory = miden_node_store::DataDirectory::load(data_directory).expect("data directory should exist"); let metrics = Box::pin(generate_blocks( + &state, account_batches, initial_accounts, if seed_public_accounts_at_genesis { @@ -279,7 +281,7 @@ pub async fn seed_store_with_readers( }, faucet, genesis_header, - &store_state, + &mut block_writer, data_directory, accounts_filepath, &signer, @@ -290,8 +292,8 @@ pub async fn seed_store_with_readers( )) .await; - // Stop the readers and report their latencies; they hold state references and the read phase - // should not include post-write quiescence. + // Stop the readers and report their latencies before stopping the store: they hold state + // references, and the read phase should not include post-write quiescence. let write_load_elapsed = start.elapsed(); stop_readers.store(true, Ordering::Relaxed); let mut read_latencies = Vec::new(); @@ -302,6 +304,10 @@ pub async fn seed_store_with_readers( report_read_latencies(&read_latencies, readers, write_load_elapsed); } + // Wait for the store to release its backing storage so callers can immediately re-load the + // state from the same data directory. + block_writer.stop(writer_task).await; + println!("Total time: {:.3} seconds", start.elapsed().as_secs_f64()); println!("{metrics}"); } @@ -309,9 +315,9 @@ pub async fn seed_store_with_readers( /// Loops `get_block_header` (latest header with MMR proof) against the state until `stop` is set, /// returning the latency of every request. /// -/// The request combines an in-memory read (opening the latest block's MMR proof, which contends -/// with the writer's lock during block application) with a database header lookup, so it exercises -/// the read path most exposed to concurrent block application. +/// The request combines an in-memory read (opening the latest block's MMR proof against the +/// snapshot's blockchain) with a database header lookup scoped to the snapshot's tip, so it +/// exercises the read path most exposed to concurrent block application. async fn read_latest_header_until( state: &Arc, stop: &AtomicBool, @@ -320,6 +326,7 @@ async fn read_latest_header_until( while !stop.load(Ordering::Relaxed) { let start = Instant::now(); let (header, proof) = state + .view() .get_block_header(None, true) .await .expect("get_block_header should succeed during seeding"); @@ -353,12 +360,13 @@ fn report_read_latencies( #[expect(clippy::too_many_arguments)] #[expect(clippy::too_many_lines)] async fn generate_blocks( + state: &State, account_batches: Vec, initial_accounts: Vec, first_account_index: u64, mut faucet: Account, genesis_header: BlockHeader, - store_state: &Arc, + block_writer: &mut BlockWriter, data_directory: DataDirectory, accounts_filepath: PathBuf, signer: &EcdsaSecretKey, @@ -439,7 +447,7 @@ async fn generate_blocks( .collect(); // create the block and send it to the store - let block_inputs = get_block_inputs(store_state, &batches, &mut metrics).await; + let block_inputs = get_block_inputs(state, &batches, &mut metrics).await; // update blocks let block_kind = if has_pending_account_creations { @@ -450,7 +458,7 @@ async fn generate_blocks( prev_block_header = apply_block( batches, block_inputs, - store_state, + block_writer, &mut metrics, signer, block_kind, @@ -461,8 +469,7 @@ async fn generate_blocks( .extend(pending_consumed_accounts.into_iter().map(|account| (account.id(), account))); // create the consume notes txs to be used in the next block - let batch_inputs = - get_batch_inputs(store_state, &prev_block_header, ¬es, &mut metrics).await; + let batch_inputs = get_batch_inputs(state, &prev_block_header, ¬es, &mut metrics).await; (pending_consumed_accounts, consume_notes_txs) = create_consume_note_txs( &prev_block_header, accounts, @@ -486,11 +493,11 @@ async fn generate_blocks( .par_chunks(TRANSACTIONS_PER_BATCH) .map(|txs| create_batch(txs, &prev_block_header)) .collect(); - let block_inputs = get_block_inputs(store_state, &batches, &mut metrics).await; + let block_inputs = get_block_inputs(state, &batches, &mut metrics).await; prev_block_header = apply_block( batches, block_inputs, - store_state, + block_writer, &mut metrics, signer, metrics::BlockKind::AccountCreation, @@ -526,11 +533,11 @@ async fn generate_blocks( let emit_note_tx = create_emit_note_tx(&prev_block_header, &mut faucet, notes.clone()); let batches = vec![create_batch(std::slice::from_ref(&emit_note_tx), &prev_block_header)]; - let block_inputs = get_block_inputs(store_state, &batches, &mut metrics).await; + let block_inputs = get_block_inputs(state, &batches, &mut metrics).await; prev_block_header = apply_block( batches, block_inputs, - store_state, + block_writer, &mut metrics, signer, metrics::BlockKind::UpdateNoteEmission, @@ -538,8 +545,7 @@ async fn generate_blocks( ) .await; - let batch_inputs = - get_batch_inputs(store_state, &prev_block_header, ¬es, &mut metrics).await; + let batch_inputs = get_batch_inputs(state, &prev_block_header, ¬es, &mut metrics).await; let accounts = selected_account_ids .iter() .map(|account_id| { @@ -562,11 +568,11 @@ async fn generate_blocks( .par_chunks(TRANSACTIONS_PER_BATCH) .map(|txs| create_batch(txs, &prev_block_header)) .collect(); - let block_inputs = get_block_inputs(store_state, &batches, &mut metrics).await; + let block_inputs = get_block_inputs(state, &batches, &mut metrics).await; prev_block_header = apply_block( batches, block_inputs, - store_state, + block_writer, &mut metrics, signer, metrics::BlockKind::AccountUpdate, @@ -595,7 +601,7 @@ async fn generate_blocks( async fn apply_block( batches: Vec, block_inputs: BlockInputs, - store_state: &Arc, + block_writer: &mut BlockWriter, metrics: &mut SeedingMetrics, signer: &EcdsaSecretKey, block_kind: metrics::BlockKind, @@ -613,7 +619,7 @@ async fn apply_block( let ordered_batches = proposed_block.batches().clone(); let start = Instant::now(); - store_state + block_writer .apply_block_with_proving_inputs(ordered_batches, block_inputs, signed_block) .await .unwrap(); @@ -1063,7 +1069,7 @@ fn create_emit_note_tx( /// Gets the batch inputs from the store and tracks the query time on the metrics. async fn get_batch_inputs( - store_state: &Arc, + state: &State, block_ref: &BlockHeader, notes: &[Note], metrics: &mut SeedingMetrics, @@ -1071,7 +1077,8 @@ async fn get_batch_inputs( let start = Instant::now(); // Mark every note as unauthenticated, so that the store returns the inclusion proofs for all of // them - let batch_inputs = store_state + let batch_inputs = state + .view() .get_batch_inputs( [block_ref.block_num()].into_iter().collect(), notes.iter().map(|note| note.id().as_word()).collect(), @@ -1084,12 +1091,13 @@ async fn get_batch_inputs( /// Gets the block inputs from the store and tracks the query time on the metrics. async fn get_block_inputs( - store_state: &Arc, + state: &State, batches: &[ProvenBatch], metrics: &mut SeedingMetrics, ) -> BlockInputs { let start = Instant::now(); - let inputs = store_state + let inputs = state + .view() .get_block_inputs( batches.iter().flat_map(ProvenBatch::updated_accounts).collect(), batches.iter().flat_map(ProvenBatch::created_nullifiers).collect(), @@ -1111,14 +1119,28 @@ async fn get_block_inputs( inputs } -/// Loads the store state from the given data directory. +/// Loads the store state from the given data directory, detaching the block writer task. +/// +/// Intended for benches that run until process exit and never need the storage released +/// deterministically; use [`load_state`] when the writer must be joined. The write capability is +/// leaked to keep the block writer alive for the process lifetime, as before the read/write +/// split. pub async fn start_store(data_directory: PathBuf) -> Arc { - load_state(data_directory).await + let (state, block_writer, _writer_task) = load_state(data_directory).await; + std::mem::forget(block_writer); + state } -async fn load_state(data_directory: PathBuf) -> Arc { - let state = State::load(&data_directory, StorageOptions::bench()) - .await - .expect("store state should load"); - Arc::new(state) +/// Loads the store state and spawns its block writer, returning the write capability and the +/// writer's join handle. +/// +/// The writer exits once the returned [`BlockWriter`] is dropped; awaiting the handle after that +/// guarantees the backing storage has been released. +async fn load_state(data_directory: PathBuf) -> (Arc, BlockWriter, WriterTask) { + let (state, block_writer, _proof_writer, writer_task) = + State::load(&data_directory, StorageOptions::bench()) + .await + .expect("store state should load") + .start(CancellationToken::new()); + (state, block_writer, writer_task) } diff --git a/bin/stress-test/src/seeding/tests.rs b/bin/stress-test/src/seeding/tests.rs index e0be9a7b26..420b96de3c 100644 --- a/bin/stress-test/src/seeding/tests.rs +++ b/bin/stress-test/src/seeding/tests.rs @@ -170,8 +170,9 @@ async fn seed_store_persists_one_public_account_and_applies_one_map_update() { assert_eq!(account_ids.len(), 1); let account_id = AccountId::from_hex(account_ids[0]).unwrap(); - let state = load_state(data_directory).await; + let (state, block_writer, writer_task) = load_state(data_directory).await; let response = state + .view() .get_account(AccountRequest { account_id, block_num: None, @@ -198,6 +199,10 @@ async fn seed_store_persists_one_public_account_and_applies_one_map_update() { .map(|(_, value)| *value), Some(benchmark_storage_map_update_value(0, 0, 1)) ); + + // Release the backing storage before the temporary directory is deleted. + drop(state); + block_writer.stop(writer_task).await; } #[tokio::test(flavor = "multi_thread")] @@ -213,8 +218,9 @@ async fn seed_store_handles_map_larger_than_transaction_account_update_limit() { assert_eq!(account_ids.len(), 1); let account_id = AccountId::from_hex(account_ids[0]).unwrap(); - let state = load_state(data_directory).await; + let (state, block_writer, writer_task) = load_state(data_directory).await; let response = state + .view() .get_account(AccountRequest { account_id, block_num: None, @@ -223,4 +229,8 @@ async fn seed_store_handles_map_larger_than_transaction_account_update_limit() { .await .unwrap(); assert_ne!(response.witness.state_commitment(), Word::empty()); + + // Release the backing storage before the temporary directory is deleted. + drop(state); + block_writer.stop(writer_task).await; } diff --git a/bin/stress-test/src/store/mod.rs b/bin/stress-test/src/store/mod.rs index 6f9440148d..c8c0789684 100644 --- a/bin/stress-test/src/store/mod.rs +++ b/bin/stress-test/src/store/mod.rs @@ -5,7 +5,7 @@ use std::time::{Duration, Instant}; use futures::{StreamExt, stream}; use miden_node_proto::domain::account::AccountRequest; use miden_node_proto::generated::{self as proto}; -use miden_node_store::state::{Finality, State}; +use miden_node_store::state::State; use miden_node_utils::clap::StorageOptions; use miden_protocol::Word; use miden_protocol::account::AccountId; @@ -127,7 +127,8 @@ async fn get_account( let start = Instant::now(); let request = AccountRequest::try_from(request).expect("request should be valid"); - let response: proto::rpc::AccountResponse = state.get_account(request).await.unwrap().into(); + let response: proto::rpc::AccountResponse = + state.view().get_account(request).await.unwrap().into(); let duration = start.elapsed(); let details = response.details; @@ -232,7 +233,7 @@ pub async fn bench_sync_notes(data_directory: PathBuf, iterations: usize, concur let store_state = start_store(data_directory).await; // Get the latest block number to determine the range - let chain_tip = store_state.chain_tip(Finality::Committed).await.as_u32(); + let chain_tip = store_state.committed_tip().as_u32(); // each request will have `ACCOUNTS_PER_SYNC_NOTES` note tags and will be sent with block number // 0. @@ -268,6 +269,7 @@ pub async fn sync_notes( .collect::>(); let start = Instant::now(); state + .view() .sync_notes(note_tags, BlockNumber::from(0)..=BlockNumber::from(chain_tip)) .await .unwrap(); @@ -304,7 +306,7 @@ pub async fn bench_sync_nullifiers( .collect(); // Get the latest block number to determine the range - let chain_tip = store_state.chain_tip(Finality::Committed).await.as_u32(); + let chain_tip = store_state.committed_tip().as_u32(); // Get all nullifier prefixes from the store using sync_notes let mut nullifier_prefixes: Vec = vec![]; @@ -316,6 +318,7 @@ pub async fn bench_sync_nullifiers( .map(|id| u32::from(NoteTag::with_account_target(*id))) .collect(); let (blocks, last_block_checked) = store_state + .view() .sync_notes( note_tags, BlockNumber::from(current_block_num)..=BlockNumber::from(chain_tip), @@ -338,6 +341,7 @@ pub async fn bench_sync_nullifiers( note_ids.iter().take(NOTE_IDS_PER_NULLIFIERS_CHECK).copied().collect(); if !note_ids_to_fetch.is_empty() { let notes = store_state + .view() .get_notes_by_id(note_ids_to_fetch) .await .unwrap() @@ -393,6 +397,7 @@ async fn sync_nullifiers( ) -> (Duration, usize) { let start = Instant::now(); let (nullifiers, _) = state + .view() .sync_nullifiers( 16, nullifiers_prefixes, @@ -438,7 +443,7 @@ pub async fn bench_sync_transactions( let store_state = start_store(data_directory).await; // Get the latest block number to determine the range - let chain_tip = store_state.chain_tip(Finality::Committed).await.as_u32(); + let chain_tip = store_state.committed_tip().as_u32(); // each request will have `accounts_per_request` account ids and will query a range of blocks let request = |_| { @@ -512,11 +517,17 @@ pub async fn sync_transactions( block_to: u32, ) -> (Duration, proto::rpc::SyncTransactionsResponse) { let start = Instant::now(); - let (last_block_included, records) = state - .sync_transactions(account_ids, BlockNumber::from(block_from)..=BlockNumber::from(block_to)) - .await - .unwrap(); - let chain_tip = state.chain_tip(Finality::Committed).await; + let (chain_tip, (last_block_included, records)) = state + .with_view(async |view| { + view.sync_transactions( + account_ids, + BlockNumber::from(block_from)..=BlockNumber::from(block_to), + ) + .await + .map(|records| (*view.tip(), records)) + .unwrap() + }) + .await; let response = proto::rpc::SyncTransactionsResponse { pagination_info: Some(proto::rpc::PaginationInfo { chain_tip: chain_tip.as_u32(), @@ -599,7 +610,7 @@ async fn sync_transactions_paginated( pub async fn bench_sync_chain_mmr(data_directory: PathBuf, iterations: usize, concurrency: usize) { let store_state = start_store(data_directory).await; - let chain_tip = store_state.chain_tip(Finality::Committed).await.as_u32(); + let chain_tip = store_state.committed_tip().as_u32(); let request = |_| { let state = Arc::clone(&store_state); @@ -628,11 +639,13 @@ pub async fn bench_sync_chain_mmr(data_directory: PathBuf, iterations: usize, co /// - the response. async fn sync_chain_mmr(state: &Arc, current_client_block_height: u32) -> SyncChainMmrRun { let start = Instant::now(); - let chain_tip = state.chain_tip(Finality::Committed).await; state - .sync_chain_mmr(BlockNumber::from(current_client_block_height)..=chain_tip) - .await - .unwrap(); + .with_view(async |view| { + view.sync_chain_mmr(BlockNumber::from(current_client_block_height)..=*view.tip()) + .await + .unwrap(); + }) + .await; let elapsed = start.elapsed(); SyncChainMmrRun { duration: elapsed } } @@ -685,7 +698,9 @@ fn transaction_record_to_proto( pub async fn load_state(data_directory: &Path) { let start = Instant::now(); - let _state = State::load(data_directory, StorageOptions::default()).await.unwrap(); + // The writer is never started: this bench only measures load time, and dropping the un-started + // state releases the tree storage the writer owns. + let _loaded = State::load(data_directory, StorageOptions::default()).await.unwrap(); let elapsed = start.elapsed(); // Get database path and run SQL commands to count records diff --git a/crates/block-producer/src/batch_builder/mod.rs b/crates/block-producer/src/batch_builder/mod.rs index e75407df26..4a9b475077 100644 --- a/crates/block-producer/src/batch_builder/mod.rs +++ b/crates/block-producer/src/batch_builder/mod.rs @@ -46,7 +46,7 @@ pub struct BatchBuilder { /// /// If not provided, a local batch prover is used. batch_prover: BatchProver, - store: Arc, + state: Arc, } pub(crate) struct BatchIntervals { @@ -80,7 +80,7 @@ impl BatchBuilder { /// /// If no batch prover URL is provided, a local batch prover is used instead. pub fn new( - store: Arc, + state: Arc, num_workers: NonZeroUsize, batch_prover_url: Option, intervals: BatchIntervals, @@ -93,7 +93,7 @@ impl BatchBuilder { num_workers, intervals, batch_prover, - store, + state, }) } @@ -160,7 +160,7 @@ impl BatchBuilder { transactions.unauthenticated_notes.count = telemetry.unauthenticated_notes_count, ); let job = BatchJob { - store: self.store.clone(), + state: self.state.clone(), mempool, batch_prover: self.batch_prover.clone(), }; @@ -238,7 +238,7 @@ impl BatchBuilder { /// /// Recoverable errors are handled internally. Mempool poison is propagated as a fatal error. struct BatchJob { - store: Arc, + state: Arc, batch_prover: BatchProver, mempool: SharedMempool, } @@ -301,7 +301,8 @@ impl BatchJob { .map(Deref::deref) .flat_map(AuthenticatedTransaction::unauthenticated_note_ids); - self.store + self.state + .view() .get_batch_inputs( block_references.map(|(block_num, _)| block_num).collect(), unauthenticated_notes.collect(), diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index 151fb3c9ba..b95c5d9295 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -2,7 +2,7 @@ use std::ops::Deref; use std::sync::Arc; use anyhow::Context; -use miden_node_store::state::State; +use miden_node_store::state::{BlockWriter, State}; use miden_node_utils::formatting::format_array; use miden_node_utils::shutdown::CancellationToken; use miden_node_utils::spawn::spawn_blocking_in_current_span; @@ -31,23 +31,33 @@ pub struct BlockBuilder { /// The frequency at which blocks are produced. pub block_interval: Duration, - /// The store state for committing blocks. - pub store: Arc, + /// Read-only store state, used to fetch block inputs. + pub state: Arc, + + /// The store's block-write capability, used for committing blocks. + pub block_writer: BlockWriter, /// The validator RPC client for validating blocks. pub validator: BlockProducerValidatorClient, } impl BlockBuilder { - /// Creates a new [`BlockBuilder`] with the given store state and optional block prover URL. + /// Creates a new [`BlockBuilder`] with the given block-write capability and optional block + /// prover URL. /// /// If the block prover URL is not set, the block builder will use the local block prover. pub fn new( - store: Arc, + state: Arc, + block_writer: BlockWriter, validator: BlockProducerValidatorClient, block_interval: Duration, ) -> Self { - Self { block_interval, store, validator } + Self { + block_interval, + state, + block_writer, + validator, + } } /// Starts the [`BlockBuilder`], infinitely producing blocks at the configured interval. /// @@ -60,7 +70,7 @@ impl BlockBuilder { /// 3. Proving the block (this is simulated using random sleeps) /// 4. Committing the block to the store pub async fn run( - self, + mut self, mempool: SharedMempool, shutdown: CancellationToken, ) -> anyhow::Result<()> { @@ -109,7 +119,7 @@ impl BlockBuilder { target = COMPONENT, name = "block_builder.build_block", )] - async fn build_block(&self, mempool: &SharedMempool) -> Result<(), BuildBlockError> { + async fn build_block(&mut self, mempool: &SharedMempool) -> Result<(), BuildBlockError> { use futures::TryFutureExt; let selected = Self::select_block(mempool)?; @@ -123,33 +133,42 @@ impl BlockBuilder { ); let block_num = selected.block_number; - self.get_block_inputs(selected) - .inspect_ok(|inputs| { - let telemetry = inputs.telemetry(); - miden_span_record!( - block.updated_accounts.count = telemetry.updated_accounts_count, - block.erased_note_proofs.count = telemetry.erased_note_proofs_count, - ); - }) - .and_then(|inputs| self.propose_block(inputs)) - .inspect_ok(|proposed_block| { - let telemetry = proposed_block_telemetry(&proposed_block.proposed_block); - miden_span_record!( - block.nullifiers.count = telemetry.nullifiers_count, - block.output_notes.count = telemetry.output_notes_count, - block.batches.output_notes.count = telemetry.batch_output_notes_count, - block.erased_notes.count = telemetry.erased_notes_count, - ); - }) - .and_then(|proposed_block| self.build_and_validate_block(proposed_block)) - .and_then(|block_commit| self.commit_block(mempool, block_commit)) - // Handle errors by propagating the error to the root span and rolling back the block. - .inspect_err(|err| Span::current().set_error(err)) - .or_else(|err| async { - Self::rollback_block(mempool, block_num)?; - Err(err) - }) - .await + // The stages run inside one async block so that its borrows are sequential: the combinator + // chain's shared borrows of `self` end at its `.await`, after which `commit_block` may take + // `&mut self` (the block-write capability). The `?` exits only this block, so the error + // handling below still sees failures from every stage. + async { + let block_commit = self + .get_block_inputs(selected) + .inspect_ok(|inputs| { + let telemetry = inputs.telemetry(); + miden_span_record!( + block.updated_accounts.count = telemetry.updated_accounts_count, + block.erased_note_proofs.count = telemetry.erased_note_proofs_count, + ); + }) + .and_then(|inputs| self.propose_block(inputs)) + .inspect_ok(|proposed_block| { + let telemetry = proposed_block_telemetry(&proposed_block.proposed_block); + miden_span_record!( + block.nullifiers.count = telemetry.nullifiers_count, + block.output_notes.count = telemetry.output_notes_count, + block.batches.output_notes.count = telemetry.batch_output_notes_count, + block.erased_notes.count = telemetry.erased_notes_count, + ); + }) + .and_then(|proposed_block| self.build_and_validate_block(proposed_block)) + .await?; + + self.commit_block(mempool, block_commit).await + } + // Handle errors by propagating the error to the root span and rolling back the block. + .inspect_err(|err| Span::current().set_error(err)) + .or_else(|err| async { + Self::rollback_block(mempool, block_num)?; + Err(err) + }) + .await } #[miden_instrument( @@ -210,7 +229,8 @@ impl BlockBuilder { batch_iter.map(Deref::deref).flat_map(ProvenBatch::created_nullifiers); let inputs = self - .store + .state + .view() .get_block_inputs( account_ids_iter.collect(), created_nullifiers_iter.collect(), @@ -341,7 +361,7 @@ impl BlockBuilder { err, )] async fn commit_block( - &self, + &mut self, mempool: &SharedMempool, block_commit: BlockCommit, ) -> Result<(), BuildBlockError> { @@ -365,7 +385,7 @@ impl BlockBuilder { tracing::debug!(target: LOG_TARGET, transactions = %format_array(transaction_ids), "Included transactions"); } - self.store + self.block_writer .apply_block_with_proving_inputs(ordered_batches, block_inputs, signed_block) .await .map_err(StoreError::ApplyBlockFailed) diff --git a/crates/block-producer/src/mempool/tests.rs b/crates/block-producer/src/mempool/tests.rs index 890b2e38e7..e056f90ae3 100644 --- a/crates/block-producer/src/mempool/tests.rs +++ b/crates/block-producer/src/mempool/tests.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use std::time::Duration; use miden_protocol::Word; use miden_protocol::block::{BlockHeader, BlockNumber}; @@ -143,18 +144,30 @@ async fn add_transaction_traces_are_correct() { let (mut uut, _) = Mempool::for_tests(); let txs = MockProvenTxBuilder::sequential(); + let tx_id = txs[0].id().to_string(); uut.add_transaction(txs[0].clone()).unwrap(); - let span_data = rx_export.recv().await.unwrap(); - assert_eq!(span_data.name, "mempool.add_transaction"); + // The exporter is global, so skip spans leaked by tests running concurrently on other threads + // by matching on this test's transaction ID. + let span_data = tokio::time::timeout(Duration::from_secs(30), async { + loop { + let span_data = rx_export.recv().await.unwrap(); + if span_data.name == "mempool.add_transaction" + && span_data + .attributes + .iter() + .any(|kv| kv.key == "transaction.id".into() && kv.value.to_string() == tx_id) + { + break span_data; + } + } + }) + .await + .expect("span for the added transaction should be exported"); + assert!(span_data.attributes.iter().any(|kv| kv.key == "code.module.name".into() && kv.value == "miden_node_block_producer::mempool".into())); - assert!( - span_data - .attributes - .iter() - .any(|kv| kv.key == "transaction.id".into() && kv.value.to_string().starts_with("0x")) - ); + assert!(tx_id.starts_with("0x")); } // BATCH FAILED TESTS diff --git a/crates/block-producer/src/proof_scheduler.rs b/crates/block-producer/src/proof_scheduler.rs index 51fcfe36be..a6c9d4c29b 100644 --- a/crates/block-producer/src/proof_scheduler.rs +++ b/crates/block-producer/src/proof_scheduler.rs @@ -18,7 +18,7 @@ use std::time::Duration; use anyhow::Context; use miden_node_proto::BlockProofRequest; -use miden_node_store::state::{Finality, State}; +use miden_node_store::state::{ProofWriter, State}; use miden_node_utils::retry::{self, Retryable}; use miden_node_utils::shutdown::CancellationToken; use miden_node_utils::tracing::miden_instrument; @@ -108,6 +108,7 @@ impl ProofTaskJoinSet { pub(crate) async fn run( block_prover: Arc, state: Arc, + mut proof_writer: ProofWriter, mut chain_tip_rx: watch::Receiver, max_concurrent_proofs: NonZeroUsize, shutdown: CancellationToken, @@ -119,7 +120,7 @@ pub(crate) async fn run( // Next block number to schedule. Initialized from the proven tip's child so we skip // already-proven blocks on restart. - let mut next_to_prove = state.chain_tip(Finality::Proven).await.child(); + let mut next_to_prove = state.proven_tip().child(); // Completed proofs waiting to be committed in order. let mut pending: BTreeMap> = BTreeMap::new(); @@ -145,9 +146,9 @@ pub(crate) async fn run( // Drain completed proofs in ascending order so the proven tip advances without // gaps. - let mut next = state.chain_tip(Finality::Proven).await.child(); + let mut next = state.proven_tip().child(); while let Some(proof_bytes) = pending.remove(&next) { - state.apply_proof(next, proof_bytes).await?; + proof_writer.apply_proof(next, proof_bytes).await?; next = next.child(); } }, diff --git a/crates/block-producer/src/rpc_sync.rs b/crates/block-producer/src/rpc_sync.rs index bdffa59475..f193b4aa82 100644 --- a/crates/block-producer/src/rpc_sync.rs +++ b/crates/block-producer/src/rpc_sync.rs @@ -5,8 +5,8 @@ use std::time::Duration; use anyhow::Context; use miden_node_proto::clients::RpcClient; use miden_node_proto::generated::rpc::{BlockSubscriptionRequest, ProofSubscriptionRequest}; -use miden_node_store::state::{Finality, State}; -use miden_node_utils::retry::{self, Retryable}; +use miden_node_store::state::{BlockWriter, ProofWriter, State}; +use miden_node_utils::retry::{self, RetryableWithContext}; use miden_node_utils::shutdown::CancellationToken; use miden_node_utils::tasks::Tasks; use miden_node_utils::tracing::miden_instrument; @@ -130,7 +130,12 @@ impl RpcReadiness { /// Synchronizes local state from an upstream RPC service. pub struct RpcSync { + /// Read-only store state, used by both loops to read tips and subscribe to commits. pub state: Arc, + /// The store's block-write capability, consumed by the block sync loop. + pub block_writer: BlockWriter, + /// The store's proof-write capability, consumed by the proof sync loop. + pub proof_writer: ProofWriter, pub source_rpc: RpcClient, pub readiness: RpcReadiness, } @@ -141,11 +146,13 @@ impl RpcSync { let mut tasks = Tasks::new(); let block_sync = BlockSync { state: Arc::clone(&self.state), + writer: self.block_writer, source_rpc: self.source_rpc.clone(), readiness: self.readiness, }; let proof_sync = ProofSync { state: self.state, + writer: self.proof_writer, source_rpc: self.source_rpc, }; @@ -161,18 +168,28 @@ impl RpcSync { struct BlockSync { state: Arc, + writer: BlockWriter, source_rpc: RpcClient, readiness: RpcReadiness, } impl BlockSync { async fn run(self, shutdown: CancellationToken) -> anyhow::Result<()> { - let retry = (|| async { - self.sync(shutdown.clone()) - .await - .and_then(|()| Err(anyhow::anyhow!("unexpected end of stream"))) + // `sync` needs `&mut self` (it applies blocks through the writer capability), so the retry + // closure cannot lend out a borrow of a capture; thread `self` through as owned context + // instead. + let retry = (|mut sync: Self| { + let shutdown = shutdown.clone(); + async move { + let result = sync + .sync(shutdown) + .await + .and_then(|()| Err(anyhow::anyhow!("unexpected end of stream"))); + (sync, result) + } }) .retry(retry::constant(RECONNECT_DELAY, None)) + .context(self) .notify(|err, _| { warn!( target: LOG_TARGET, @@ -183,7 +200,7 @@ impl BlockSync { }); tokio::select! { - result = retry => result, + (_, result) = retry => result, () = shutdown.cancelled() => Ok(()), } } @@ -192,8 +209,8 @@ impl BlockSync { target = COMPONENT, err, )] - async fn sync(&self, shutdown: CancellationToken) -> anyhow::Result<()> { - let local_tip = self.state.chain_tip(Finality::Committed).await; + async fn sync(&mut self, shutdown: CancellationToken) -> anyhow::Result<()> { + let local_tip = self.state.committed_tip(); let mut client = self.source_rpc.clone(); let upstream_tip = BlockNumber::from(client.status(tonic::Request::new(())).await?.into_inner().chain_tip); @@ -219,9 +236,9 @@ impl BlockSync { let upstream_tip = BlockNumber::from(event.committed_chain_tip); let block = SignedBlock::read_from_bytes(&event.block) .context("failed to deserialize block from upstream")?; - self.state.apply_block(block).await?; + self.writer.apply_block(block).await?; - let local_tip = self.state.chain_tip(Finality::Committed).await; + let local_tip = self.state.committed_tip(); self.readiness.update(upstream_tip, local_tip).await; } } @@ -229,6 +246,7 @@ impl BlockSync { struct ProofSync { state: Arc, + writer: ProofWriter, source_rpc: RpcClient, } @@ -240,12 +258,21 @@ impl ProofSync { /// behind, but that is acceptable — there is no value in streaming proofs for blocks that have not /// yet been applied. async fn run(self, shutdown: CancellationToken) -> anyhow::Result<()> { - let retry = (|| async { - self.sync(shutdown.clone()) - .await - .and_then(|()| Err(anyhow::anyhow!("unexpected end of stream"))) + // `sync` needs `&mut self` (it commits proofs through the writer capability), so the retry + // closure cannot lend out a borrow of a capture; thread `self` through as owned context + // instead. + let retry = (|mut sync: Self| { + let shutdown = shutdown.clone(); + async move { + let result = sync + .sync(shutdown) + .await + .and_then(|()| Err(anyhow::anyhow!("unexpected end of stream"))); + (sync, result) + } }) .retry(retry::constant(RECONNECT_DELAY, None)) + .context(self) .notify(|err, _| { warn!( target: LOG_TARGET, @@ -256,14 +283,14 @@ impl ProofSync { }); tokio::select! { - result = retry => result, + (_, result) = retry => result, () = shutdown.cancelled() => Ok(()), } } - async fn sync(&self, shutdown: CancellationToken) -> anyhow::Result<()> { + async fn sync(&mut self, shutdown: CancellationToken) -> anyhow::Result<()> { // Subscribe from next proven tip. - let starting_block = self.state.chain_tip(Finality::Proven).await.child(); + let starting_block = self.state.proven_tip().child(); info!( target: LOG_TARGET, block_from = %starting_block, @@ -302,7 +329,7 @@ impl ProofSync { }, } - self.state.apply_proof(block_num, event.proof).await?; + self.writer.apply_proof(block_num, event.proof).await?; expected = expected.child(); } diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index 9f9b4bcc49..65fb6d14f3 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use std::time::Duration; use anyhow::Result; -use miden_node_store::state::{Finality, State}; +use miden_node_store::state::{BlockWriter, ProofWriter, State}; use miden_node_utils::formatting::{format_input_notes, format_output_notes}; use miden_node_utils::shutdown::CancellationToken; use miden_node_utils::tasks::Tasks; @@ -69,8 +69,12 @@ impl BlockProducerApiConfig { /// /// Specifies how to connect to the batch prover and block prover components. pub struct Sequencer { - /// The store state shared with the block producer. - pub store: Arc, + /// The read-only store state shared with the block producer. + pub state: Arc, + /// The store's block-write capability, handed to the block builder. + pub block_writer: BlockWriter, + /// The store's proof-write capability, handed to the proof scheduler. + pub proof_writer: ProofWriter, /// The addresses of the validator components. pub validator_urls: Vec, /// The request timeout for calls to the validator components. @@ -102,19 +106,24 @@ pub struct Sequencer { impl Sequencer { /// Spawns the sequencer tasks and returns its in-process API. - pub async fn spawn(self, shutdown: CancellationToken) -> Result { + pub fn spawn(self, shutdown: CancellationToken) -> Result { tracing::info!(target: LOG_TARGET, "Initializing sequencer"); - let store = self.store; + let state = self.state; let validator = BlockProducerValidatorClient::new(self.validator_urls.clone(), self.validator_timeout)?; - let chain_tip = store.chain_tip(Finality::Committed).await; + let chain_tip = state.committed_tip(); tracing::info!(target: LOG_TARGET, "Sequencer initialized"); - let block_builder = BlockBuilder::new(Arc::clone(&store), validator, self.block_interval); + let block_builder = BlockBuilder::new( + Arc::clone(&state), + self.block_writer, + validator, + self.block_interval, + ); let batch_intervals = BatchIntervals::derive_from(self.block_interval, self.batch_interval); let batch_builder = BatchBuilder::new( - Arc::clone(&store), + Arc::clone(&state), self.batch_workers, self.batch_prover_url, batch_intervals, @@ -125,13 +134,13 @@ impl Sequencer { mempool_tx_capacity: self.mempool_tx_capacity, }; let mempool = Mempool::shared(chain_tip, api_config.mempool_config()); - let api = BlockProducerApi::from_shared_mempool(mempool.clone(), store, shutdown.clone()); + let api = BlockProducerApi::from_shared_mempool(mempool.clone(), state, shutdown.clone()); let block_prover = if let Some(url) = self.block_prover_url { Arc::new(BlockProver::remote(url)?) } else { Arc::new(BlockProver::local()) }; - let chain_tip_rx = api.store.subscribe_committed_tip(); + let chain_tip_rx = api.state.subscribe_committed_tip(); // Spawn batch builder, block builder, and proof scheduler. The builders communicate // indirectly via a shared mempool. @@ -151,12 +160,14 @@ impl Sequencer { async { block_builder.run(mempool, shutdown).await } }); tasks.spawn("proof-scheduler", { - let store = Arc::clone(&api.store); + let state = Arc::clone(&api.state); + let proof_writer = self.proof_writer; let shutdown = shutdown.clone(); async move { proof_scheduler::run( block_prover, - store, + state, + proof_writer, chain_tip_rx, self.max_concurrent_proofs, shutdown, @@ -168,14 +179,6 @@ impl Sequencer { Ok(SequencerHandle { api, task }) } - - /// Serves the sequencer tasks. - /// - /// Executes in place (i.e. not spawned) and will run indefinitely until a fatal error is - /// encountered. - pub async fn serve(self) -> anyhow::Result<()> { - self.spawn(CancellationToken::new()).await?.wait().await - } } /// Running sequencer tasks plus the API used to submit work to them. @@ -210,7 +213,7 @@ pub struct BlockProducerApi { /// the block-producers usage of the mempool. mempool: Arc>, - store: Arc, + state: Arc, /// Cached mempool statistics that are updated periodically to avoid locking the mempool for /// each status request. @@ -238,17 +241,17 @@ pub struct BlockProducerStatus { impl BlockProducerApi { /// Creates an API backed by a fresh mempool. - pub fn new(store: Arc, chain_tip: BlockNumber, config: BlockProducerApiConfig) -> Self { + pub fn new(state: Arc, chain_tip: BlockNumber, config: BlockProducerApiConfig) -> Self { Self::from_shared_mempool( Mempool::shared(chain_tip, config.mempool_config()), - store, + state, CancellationToken::new(), ) } fn from_shared_mempool( mempool: SharedMempool, - store: Arc, + state: Arc, shutdown: CancellationToken, ) -> Self { let cached_mempool_stats = mempool @@ -257,7 +260,7 @@ impl BlockProducerApi { .unwrap_or_default(); let api = Self { mempool: Arc::new(Mutex::new(mempool)), - store, + state, cached_mempool_stats: Arc::new(RwLock::new(cached_mempool_stats)), }; api.spawn_mempool_stats_updater(shutdown); @@ -328,7 +331,7 @@ impl BlockProducerApi { tracing::debug!(target: COMPONENT, proof = ?tx.proof()); // Authenticate against the local store, then add to the mempool. - let inputs = get_tx_inputs(&self.store, &tx) + let inputs = get_tx_inputs(&self.state, &tx) .await .map_err(MempoolSubmissionError::StoreStateReadFailed)?; // SAFETY: we assume that the rpc component has verified the transaction proof already. @@ -373,7 +376,7 @@ impl BlockProducerApi { let mut inputs = Vec::with_capacity(batch.transactions().len()); for tx in batch.transactions() { inputs.push( - get_tx_inputs(&self.store, tx) + get_tx_inputs(&self.state, tx) .await .map_err(MempoolSubmissionError::StoreStateReadFailed)?, ); diff --git a/crates/block-producer/src/server/tests.rs b/crates/block-producer/src/server/tests.rs index 5ee80cd967..ed3764dc18 100644 --- a/crates/block-producer/src/server/tests.rs +++ b/crates/block-producer/src/server/tests.rs @@ -1,10 +1,8 @@ use std::num::NonZeroUsize; -use std::sync::Arc; use std::time::Duration; use miden_node_store::GenesisState; use miden_node_store::state::State; -use miden_node_utils::clap::StorageOptions; use miden_node_utils::fee::test_fee_params; use miden_protocol::block::{BlockNumber, ValidatorKeys}; use miden_protocol::testing::random_secret_key::random_secret_key; @@ -23,10 +21,12 @@ use crate::{ async fn block_producer_starts_with_store_state() { let data_directory = tempfile::tempdir().expect("tempdir should be created"); bootstrap_store(data_directory.path()); - let store = load_state(data_directory.path()).await; + let (state, block_writer, proof_writer) = State::for_tests(data_directory.path()).await; let block_producer = Sequencer { - store, + state, + block_writer, + proof_writer, validator_urls: vec![Url::parse("http://127.0.0.1:0").unwrap()], validator_timeout: DEFAULT_VALIDATOR_TIMEOUT, batch_prover_url: None, @@ -40,7 +40,6 @@ async fn block_producer_starts_with_store_state() { batch_workers: DEFAULT_BATCH_WORKERS, } .spawn(miden_node_utils::shutdown::CancellationToken::new()) - .await .unwrap(); let status = block_producer.api().status().await; @@ -61,8 +60,3 @@ fn bootstrap_store(path: &std::path::Path) { State::bootstrap(genesis_block, path).expect("store should bootstrap"); } - -async fn load_state(path: &std::path::Path) -> Arc { - let state = State::load(path, StorageOptions::default()).await.expect("state should load"); - Arc::new(state) -} diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index c00f7e31d0..dfc6d6a76c 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -7,7 +7,7 @@ use miden_node_proto::decode; use miden_node_proto::decode::GrpcDecodeExt; use miden_node_proto::errors::ConversionError; use miden_node_proto::generated::sequencer; -use miden_node_store::state::{Finality, State, TransactionInputs as StoreTransactionInputs}; +use miden_node_store::state::{State, TransactionInputs as StoreTransactionInputs}; use miden_node_utils::formatting::format_opt; use miden_node_utils::tracing::miden_instrument; use miden_protocol::Word; @@ -184,14 +184,18 @@ pub async fn get_tx_inputs( let unauthenticated_note_commitments = proven_tx.unauthenticated_notes().map(|header| header.id().as_word()).collect(); - let store_inputs = state - .get_transaction_inputs( - proven_tx.account_id(), - &nullifiers, - unauthenticated_note_commitments, - ) - .await - .map_err(StoreError::GetTransactionInputsFailed)?; + let (current_block_height, store_inputs) = state + .with_view(async |view| { + view.get_transaction_inputs( + proven_tx.account_id(), + &nullifiers, + unauthenticated_note_commitments, + ) + .await + .map(|inputs| (view.tip(), inputs)) + .map_err(StoreError::GetTransactionInputsFailed) + }) + .await?; if !store_inputs.new_account_id_prefix_is_unique.unwrap_or(true) { debug_assert!( @@ -201,11 +205,10 @@ pub async fn get_tx_inputs( return Err(StoreError::DuplicateAccountIdPrefix(proven_tx.account_id())); } - let current_block_height = state.chain_tip(Finality::Committed).await; let tx_inputs = TransactionInputs::from_store_inputs( proven_tx.account_id(), store_inputs, - current_block_height, + *current_block_height, ); tracing::debug!(target: LOG_TARGET, tx_inputs = %tx_inputs, "Transaction inputs"); diff --git a/crates/rpc/src/server/api.rs b/crates/rpc/src/server/api.rs index f288e7a4dd..cbcefc811e 100644 --- a/crates/rpc/src/server/api.rs +++ b/crates/rpc/src/server/api.rs @@ -1,5 +1,4 @@ use std::num::NonZeroUsize; -use std::ops::RangeInclusive; use std::sync::{Arc, LazyLock}; use std::time::Duration; @@ -10,7 +9,7 @@ use miden_node_proto::domain::block::InvalidBlockRange; use miden_node_proto::generated::rpc::MempoolStats as ProtoMempoolStats; use miden_node_proto::generated::rpc::api_server::Api; use miden_node_proto::generated::{self as proto}; -use miden_node_store::state::{Finality, State}; +use miden_node_store::state::State; use miden_node_store::{DatabaseError, GetBlockHeaderError}; use miden_node_utils::limiter::{ QueryParamAccountIdLimit, @@ -112,7 +111,7 @@ impl From for RpcInvalidBlockRange { // ================================================================================================ pub struct RpcService { - store: Arc, + state: Arc, mode: RpcMode, ntx_builder: Option, network_tx_auth: Option, @@ -125,14 +124,14 @@ pub struct RpcService { impl RpcService { pub(crate) fn new( - store: Arc, + state: Arc, mode: RpcMode, ntx_builder: Option, commitment_cache_capacity: NonZeroUsize, network_tx_auth: Option, ) -> Self { Self { - store, + state, mode, ntx_builder, network_tx_auth, @@ -146,8 +145,8 @@ impl RpcService { /// Sets the genesis commitment, returning an error if it is already set. /// - /// Required since the store client is used to fetch the `genesis_commitment` after - /// `RpcService` construction. + /// Required since the genesis header is fetched through the store state after `RpcService` + /// construction. pub fn set_genesis_commitment(&mut self, commitment: Word) -> anyhow::Result<()> { if self.genesis_commitment.is_some() { return Err(anyhow::anyhow!("genesis commitment already set")); @@ -203,7 +202,8 @@ impl RpcService { } let header = self - .store + .state + .view() .get_block_header(Some(block), false) .await .map_err(get_block_header_error_to_status)? @@ -233,25 +233,6 @@ impl RpcService { Ok(()) } - /// Fetches the committed chain tip and ensures the requested range does not extend beyond it. - /// - /// Returns the chain tip so callers can reuse it (e.g. in the response's pagination info) - /// without issuing a second query. - async fn range_bounds_check( - &self, - range: &RangeInclusive, - ) -> Result { - let chain_tip = self.store.chain_tip(Finality::Committed).await; - if *range.end() > chain_tip { - return Err(Status::invalid_argument(format!( - "block_to ({}) is greater than chain tip ({chain_tip})", - range.end() - ))); - } - - Ok(chain_tip) - } - /// Errors if any of `candidate_ids` is classified as a network account by the store. Callers /// should pre-filter to post-deployment, public-account ids; `Ok(())` on empty. async fn reject_if_any_network_accounts( @@ -264,7 +245,7 @@ impl RpcService { } let network_accounts = - self.store.filter_network_accounts(&account_ids).await.map_err(|err| { + self.state.view().filter_network_accounts(&account_ids).await.map_err(|err| { Status::internal(format!("network-account classification failed: {err}")) })?; @@ -310,6 +291,7 @@ fn database_error_to_status(err: &DatabaseError) -> Status { | DatabaseError::AccountsNotFoundInDb(_) | DatabaseError::AccountNotPublic(_) => Status::not_found(message), DatabaseError::TransactionPageExceedsPayloadLimit { .. } => Status::out_of_range(message), + DatabaseError::RangeBeyondTip(_) => Status::invalid_argument(message), _ => Status::internal(message), } } diff --git a/crates/rpc/src/server/api/get_account.rs b/crates/rpc/src/server/api/get_account.rs index 592160f27a..2e38d528f1 100644 --- a/crates/rpc/src/server/api/get_account.rs +++ b/crates/rpc/src/server/api/get_account.rs @@ -57,8 +57,12 @@ impl proto::server::rpc_api::GetAccount for RpcService { validate_storage_request(&details.storage_request)?; } - let account_data = - self.store.get_account(request).await.map_err(get_account_error_to_status)?; + let account_data = self + .state + .view() + .get_account(request) + .await + .map_err(get_account_error_to_status)?; Ok(account_data) } } diff --git a/crates/rpc/src/server/api/get_block_by_number.rs b/crates/rpc/src/server/api/get_block_by_number.rs index 21838f47ce..49af795256 100644 --- a/crates/rpc/src/server/api/get_block_by_number.rs +++ b/crates/rpc/src/server/api/get_block_by_number.rs @@ -37,12 +37,12 @@ impl proto::server::rpc_api::GetBlockByNumber for RpcService { let block_num = BlockNumber::from(request.block_num); let block = self - .store + .state .load_block(block_num) .await .map_err(|err| database_error_to_status(&err))?; let proof = if request.include_proof.unwrap_or_default() { - self.store + self.state .load_proof(block_num) .await .map_err(|err| database_error_to_status(&err))? diff --git a/crates/rpc/src/server/api/get_block_header_by_number.rs b/crates/rpc/src/server/api/get_block_header_by_number.rs index e138ccf0c7..6d1525a80f 100644 --- a/crates/rpc/src/server/api/get_block_header_by_number.rs +++ b/crates/rpc/src/server/api/get_block_header_by_number.rs @@ -37,7 +37,8 @@ impl proto::server::rpc_api::GetBlockHeaderByNumber for RpcService { let block_num = request.block_num.map(BlockNumber::from); let (block_header, mmr_proof) = self - .store + .state + .view() .get_block_header(block_num, request.include_mmr_proof.unwrap_or(false)) .await .map_err(super::get_block_header_error_to_status)?; diff --git a/crates/rpc/src/server/api/get_note_script_by_root.rs b/crates/rpc/src/server/api/get_note_script_by_root.rs index cbcff75d15..1b955b7d07 100644 --- a/crates/rpc/src/server/api/get_note_script_by_root.rs +++ b/crates/rpc/src/server/api/get_note_script_by_root.rs @@ -42,7 +42,8 @@ impl proto::server::rpc_api::GetNoteScriptByRoot for RpcService { debug!(target: LOG_TARGET, "Getting note script by root"); let script = self - .store + .state + .view() .get_note_script_by_root(root) .await .map_err(|err| database_error_to_status(&err))?; diff --git a/crates/rpc/src/server/api/get_notes_by_id.rs b/crates/rpc/src/server/api/get_notes_by_id.rs index 1fe77bed86..bcbc6bcb90 100644 --- a/crates/rpc/src/server/api/get_notes_by_id.rs +++ b/crates/rpc/src/server/api/get_notes_by_id.rs @@ -44,7 +44,8 @@ impl proto::server::rpc_api::GetNotesById for RpcService { let note_ids: Vec = note_ids.into_iter().map(NoteId::from_raw).collect(); let notes = self - .store + .state + .view() .get_notes_by_id(note_ids) .await .map_err(|err| database_error_to_status(&err))? diff --git a/crates/rpc/src/server/api/status.rs b/crates/rpc/src/server/api/status.rs index 55ad02880e..7ed55a2ea2 100644 --- a/crates/rpc/src/server/api/status.rs +++ b/crates/rpc/src/server/api/status.rs @@ -3,7 +3,7 @@ use miden_node_proto::generated as proto; use miden_node_utils::tracing::miden_instrument; use tracing::debug; -use super::{Finality, ProtoMempoolStats, Request, RpcMode, RpcService}; +use super::{ProtoMempoolStats, Request, RpcMode, RpcService}; use crate::{COMPONENT, LOG_TARGET}; #[tonic::async_trait] @@ -47,7 +47,7 @@ impl proto::server::rpc_api::Status for RpcService { Ok(proto::rpc::RpcStatus { version: env!("CARGO_PKG_VERSION").to_string(), - chain_tip: self.store.chain_tip(Finality::Committed).await.as_u32(), + chain_tip: self.state.committed_tip().as_u32(), block_producer: block_producer_status.or(Some(proto::rpc::BlockProducerStatus { status: "unreachable".to_string(), version: "-".to_string(), diff --git a/crates/rpc/src/server/api/submit_proven_tx.rs b/crates/rpc/src/server/api/submit_proven_tx.rs index 3aa486e8c1..80a0273f9d 100644 --- a/crates/rpc/src/server/api/submit_proven_tx.rs +++ b/crates/rpc/src/server/api/submit_proven_tx.rs @@ -168,7 +168,7 @@ impl RpcService { request: proto::transaction::ProvenTransaction, rebuilt_tx: ProvenTransaction, ) -> tonic::Result { - let tx_inputs = get_tx_inputs(&self.store, &rebuilt_tx).await.map_err(|err| { + let tx_inputs = get_tx_inputs(&self.state, &rebuilt_tx).await.map_err(|err| { Status::internal(err.as_report_context("failed to get transaction inputs")) })?; diff --git a/crates/rpc/src/server/api/submit_proven_tx_batch.rs b/crates/rpc/src/server/api/submit_proven_tx_batch.rs index 48c1c5d70a..944e102d6c 100644 --- a/crates/rpc/src/server/api/submit_proven_tx_batch.rs +++ b/crates/rpc/src/server/api/submit_proven_tx_batch.rs @@ -166,7 +166,7 @@ impl RpcService { let mut auth_inputs = Vec::with_capacity(proposed_batch.transactions().len()); for tx in proposed_batch.transactions() { - let inputs = get_tx_inputs(&self.store, tx).await.map_err(|err| { + let inputs = get_tx_inputs(&self.state, tx).await.map_err(|err| { Status::internal(err.as_report_context("failed to authenticate transaction")) })?; auth_inputs.push(inputs.into()); diff --git a/crates/rpc/src/server/api/subscription/stream/mod.rs b/crates/rpc/src/server/api/subscription/stream/mod.rs index 77e9406214..5b88c86057 100644 --- a/crates/rpc/src/server/api/subscription/stream/mod.rs +++ b/crates/rpc/src/server/api/subscription/stream/mod.rs @@ -61,14 +61,14 @@ impl SubscriptionStream { from: BlockNumber, client_ip: Option, ) -> tonic::Result { - let store = Arc::clone(&rpc.store); + let store = Arc::clone(&rpc.state); Self::create( from, client_ip, Arc::clone(&rpc.subscription_ban), Arc::clone(&rpc.block_subscription_semaphore), - rpc.store.subscribe_committed_tip(), + rpc.state.subscribe_committed_tip(), move |block| { let store = Arc::clone(&store); async move { store.load_block(block).await } @@ -82,14 +82,14 @@ impl SubscriptionStream { from: BlockNumber, client_ip: Option, ) -> tonic::Result { - let store = Arc::clone(&rpc.store); + let store = Arc::clone(&rpc.state); Self::create( from, client_ip, Arc::clone(&rpc.subscription_ban), Arc::clone(&rpc.proof_subscription_semaphore), - rpc.store.subscribe_proven_tip(), + rpc.state.subscribe_proven_tip(), move |block| { let store = Arc::clone(&store); async move { store.load_proof(block).await } diff --git a/crates/rpc/src/server/api/sync_account_storage_maps.rs b/crates/rpc/src/server/api/sync_account_storage_maps.rs index 58b3482d97..7ad0fcc6e3 100644 --- a/crates/rpc/src/server/api/sync_account_storage_maps.rs +++ b/crates/rpc/src/server/api/sync_account_storage_maps.rs @@ -57,12 +57,15 @@ impl proto::server::rpc_api::SyncAccountStorageMaps for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let chain_tip = self.range_bounds_check(&block_range).await?; - let storage_maps_page = self - .store - .sync_account_storage_maps(account_id, block_range) - .await - .map_err(|err| database_error_to_status(&err))?; + let (chain_tip, storage_maps_page) = self + .state + .with_view(async |view| { + view.sync_account_storage_maps(account_id, block_range) + .await + .map(|page| (view.tip(), page)) + .map_err(|err| database_error_to_status(&err)) + }) + .await?; let updates = storage_maps_page .values .into_iter() diff --git a/crates/rpc/src/server/api/sync_account_vault.rs b/crates/rpc/src/server/api/sync_account_vault.rs index 0cbe05c883..44f238871b 100644 --- a/crates/rpc/src/server/api/sync_account_vault.rs +++ b/crates/rpc/src/server/api/sync_account_vault.rs @@ -57,12 +57,15 @@ impl proto::server::rpc_api::SyncAccountVault for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let chain_tip = self.range_bounds_check(&block_range).await?; - let (last_included_block, updates) = self - .store - .sync_account_vault(account_id, block_range) - .await - .map_err(|err| database_error_to_status(&err))?; + let (chain_tip, (last_included_block, updates)) = self + .state + .with_view(async |view| { + view.sync_account_vault(account_id, block_range) + .await + .map(|updates| (view.tip(), updates)) + .map_err(|err| database_error_to_status(&err)) + }) + .await?; let updates = updates .into_iter() .map(|update| { diff --git a/crates/rpc/src/server/api/sync_chain_mmr.rs b/crates/rpc/src/server/api/sync_chain_mmr.rs index f075db7fc4..1f3cea4f65 100644 --- a/crates/rpc/src/server/api/sync_chain_mmr.rs +++ b/crates/rpc/src/server/api/sync_chain_mmr.rs @@ -1,10 +1,11 @@ use miden_node_proto::generated as proto; +use miden_node_store::StateSyncError; use miden_node_utils::tracing::miden_instrument; use miden_protocol::block::BlockNumber; use tonic::Status; use tracing::debug; -use super::{Finality, RpcService}; +use super::RpcService; use crate::{COMPONENT, LOG_TARGET}; #[tonic::async_trait] @@ -38,11 +39,15 @@ impl proto::server::rpc_api::SyncChainMmr for RpcService { debug!(target: LOG_TARGET, "Syncing chain MMR"); let current_client_block_height = BlockNumber::from(request.current_client_block_height); + + // Read the target tip before creating the view: tips are monotonic and each committed tip + // is published after its snapshot, so the view below is guaranteed to be able to serve + // `sync_target`. let sync_target = match request.finality_level() { proto::rpc::FinalityLevel::Committed | proto::rpc::FinalityLevel::Unspecified => { - self.store.chain_tip(Finality::Committed).await + self.state.committed_tip() }, - proto::rpc::FinalityLevel::Proven => self.store.chain_tip(Finality::Proven).await, + proto::rpc::FinalityLevel::Proven => self.state.proven_tip(), }; if current_client_block_height > sync_target { @@ -52,11 +57,13 @@ impl proto::server::rpc_api::SyncChainMmr for RpcService { } let block_range = current_client_block_height..=sync_target; - let (mmr_delta, block_header, block_signatures) = self - .store - .sync_chain_mmr(block_range.clone()) - .await - .map_err(|err| Status::internal(err.to_string()))?; + let (mmr_delta, block_header, block_signatures) = + self.state.view().sync_chain_mmr(block_range.clone()).await.map_err( + |err| match err { + StateSyncError::RangeBeyondTip(_) => Status::invalid_argument(err.to_string()), + _ => Status::internal(err.to_string()), + }, + )?; Ok(proto::rpc::SyncChainMmrResponse { block_range: Some(proto::rpc::BlockRange { diff --git a/crates/rpc/src/server/api/sync_notes.rs b/crates/rpc/src/server/api/sync_notes.rs index 2125185b0e..48df263de3 100644 --- a/crates/rpc/src/server/api/sync_notes.rs +++ b/crates/rpc/src/server/api/sync_notes.rs @@ -46,13 +46,15 @@ impl proto::server::rpc_api::SyncNotes for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let chain_tip = self.range_bounds_check(&block_range).await?; - - let (results, last_block_checked) = self - .store - .sync_notes(request.note_tags, block_range) - .await - .map_err(note_sync_error_to_status)?; + let (chain_tip, (results, last_block_checked)) = self + .state + .with_view(async |view| { + view.sync_notes(request.note_tags, block_range) + .await + .map(|notes| (view.tip(), notes)) + .map_err(note_sync_error_to_status) + }) + .await?; let blocks = results .into_iter() .map(|(state, mmr_proof)| proto::rpc::sync_notes_response::NoteSyncBlock { @@ -116,7 +118,7 @@ fn note_sync_error_to_status(err: NoteSyncError) -> Status { match err { NoteSyncError::DatabaseError(err) => super::database_error_to_status(&err), NoteSyncError::InvalidBlockRange(_) - | NoteSyncError::FutureBlock { .. } + | NoteSyncError::RangeBeyondTip(_) | NoteSyncError::DeserializationFailed(_) => Status::invalid_argument(message), NoteSyncError::UnderlyingDatabaseError(_) | NoteSyncError::EmptyBlockHeadersTable diff --git a/crates/rpc/src/server/api/sync_nullifiers.rs b/crates/rpc/src/server/api/sync_nullifiers.rs index 9c929f3ed3..82b6e7dc26 100644 --- a/crates/rpc/src/server/api/sync_nullifiers.rs +++ b/crates/rpc/src/server/api/sync_nullifiers.rs @@ -57,13 +57,15 @@ impl proto::server::rpc_api::SyncNullifiers for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let chain_tip = self.range_bounds_check(&block_range).await?; - - let (nullifiers, block_num) = self - .store - .sync_nullifiers(request.prefix_len, request.nullifiers, block_range) - .await - .map_err(|err| database_error_to_status(&err))?; + let (chain_tip, (nullifiers, block_num)) = self + .state + .with_view(async |view| { + view.sync_nullifiers(request.prefix_len, request.nullifiers, block_range) + .await + .map(|nullifiers| (view.tip(), nullifiers)) + .map_err(|err| database_error_to_status(&err)) + }) + .await?; let nullifiers = nullifiers .into_iter() .map(|nullifier_info| proto::rpc::sync_nullifiers_response::NullifierUpdate { diff --git a/crates/rpc/src/server/api/sync_transactions.rs b/crates/rpc/src/server/api/sync_transactions.rs index 985ecf2395..984e9d7c84 100644 --- a/crates/rpc/src/server/api/sync_transactions.rs +++ b/crates/rpc/src/server/api/sync_transactions.rs @@ -61,13 +61,16 @@ impl proto::server::rpc_api::SyncTransactions for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let chain_tip = self.range_bounds_check(&block_range).await?; let account_ids = read_account_ids::(request.account_ids)?; - let (last_block_included, transaction_records_db) = self - .store - .sync_transactions(account_ids, block_range) - .await - .map_err(|err| database_error_to_status(&err))?; + let (chain_tip, (last_block_included, transaction_records_db)) = self + .state + .with_view(async |view| { + view.sync_transactions(account_ids, block_range) + .await + .map(|records| (view.tip(), records)) + .map_err(|err| database_error_to_status(&err)) + }) + .await?; let transactions = transaction_records_db.into_iter().map(transaction_record_to_proto).collect(); diff --git a/crates/rpc/src/server/mod.rs b/crates/rpc/src/server/mod.rs index 22beebe61c..e2a1fc585c 100644 --- a/crates/rpc/src/server/mod.rs +++ b/crates/rpc/src/server/mod.rs @@ -13,7 +13,7 @@ use miden_node_proto::clients::{ }; use miden_node_proto::server::{rpc_api, sequencer_api}; use miden_node_proto_build::rpc_api_descriptor; -use miden_node_store::state::{Finality, State}; +use miden_node_store::state::{BlockWriter, ProofWriter, State}; use miden_node_utils::clap::{GrpcOptionsExternal, GrpcOptionsInternal}; use miden_node_utils::cors::cors_for_grpc_web_layer; use miden_node_utils::grpc; @@ -45,8 +45,13 @@ mod health; /// It uses the supplied store state and mode-specific submission handling. pub struct Rpc { pub listener: TcpListener, - pub store: Arc, + pub state: Arc, pub mode: RpcMode, + /// Store write capabilities consumed by the full-node sync loop. + /// + /// Must be provided in full-node mode and omitted in sequencer mode. The RPC service itself + /// only ever reads through `state`; these are passed through untouched to the sync tasks. + pub sync_writers: Option<(BlockWriter, ProofWriter)>, pub ntx_builder: Option, pub grpc_options: GrpcOptionsExternal, pub network_tx_auth: Option, @@ -188,7 +193,7 @@ impl Rpc { let endpoint = self.listener.local_addr().context("failed to read RPC listen address")?; let mode = self.mode.as_str(); let mut api = api::RpcService::new( - self.store.clone(), + self.state.clone(), self.mode.clone(), self.ntx_builder.clone(), NonZeroUsize::new(1_000_000).unwrap(), @@ -216,7 +221,7 @@ impl Rpc { tonic_health::ServingStatus::Serving, ) .await; - let chain_tip = self.store.chain_tip(Finality::Committed).await; + let chain_tip = self.state.committed_tip(); log_node_ready(mode, endpoint, chain_tip); }, RpcMode::FullNode { source_rpc, readiness_threshold, .. } => { @@ -227,10 +232,15 @@ impl Rpc { ) .await; let readiness = RpcReadiness::new(health_reporter, readiness_threshold); + let (block_writer, proof_writer) = self.sync_writers.context( + "full-node RPC requires the store write capabilities for its sync loop", + )?; tasks.spawn( "RPC sync", RpcSync { - state: Arc::clone(&self.store), + state: Arc::clone(&self.state), + block_writer, + proof_writer, source_rpc: *source_rpc, readiness, } diff --git a/crates/rpc/src/tests.rs b/crates/rpc/src/tests.rs index fa51565966..4c86cfd29a 100644 --- a/crates/rpc/src/tests.rs +++ b/crates/rpc/src/tests.rs @@ -22,7 +22,7 @@ use miden_node_proto::generated::{self as proto}; use miden_node_proto::server::{ntx_builder_api, rpc_api, validator_api}; use miden_node_store::genesis::config::GenesisConfig; use miden_node_store::state::State; -use miden_node_utils::clap::{GrpcOptionsExternal, StorageOptions}; +use miden_node_utils::clap::GrpcOptionsExternal; use miden_node_utils::limiter::{ QueryParamAccountIdLimit, QueryParamLimiter, @@ -94,7 +94,7 @@ impl TestStore { async fn start() -> Self { let data_directory = new_tempdir(); let genesis_commitment = Self::bootstrap(&data_directory); - let state = load_state(&data_directory).await; + let (state, ..) = State::for_tests(&data_directory).await; Self { state, genesis_commitment, @@ -115,11 +115,6 @@ impl TestStore { } } -async fn load_state(path: &std::path::Path) -> Arc { - let state = State::load(path, StorageOptions::default()).await.expect("state should load"); - Arc::new(state) -} - /// Byte offset of the account delta commitment in serialized `ProvenTransaction`. Layout: /// `AccountId` (15) + `initial_commitment` (32) + `final_commitment` (32) = 79 const DELTA_COMMITMENT_BYTE_OFFSET: usize = 15 + 32 + 32; @@ -631,8 +626,8 @@ async fn start_source_rpc( let store = TestStore::start().await; let block_producer_dir = new_tempdir(); TestStore::bootstrap(&block_producer_dir); - let block_producer_state = load_state(&block_producer_dir).await; - let store_state = Arc::clone(&store.state); + let (block_producer_state, ..) = State::for_tests(&block_producer_dir).await; + let state = Arc::clone(&store.state); let listener = TcpListener::bind("127.0.0.1:0").await.expect("Failed to bind source RPC"); let addr = listener.local_addr().expect("Failed to get source RPC address"); @@ -644,7 +639,7 @@ async fn start_source_rpc( BlockProducerApiConfig::default(), ); let source_rpc = RpcService::new( - store_state, + state, RpcMode::sequencer(block_producer, ValidatorClients::new(vec![validator]).unwrap()), Some(ntx_builder), NonZeroUsize::new(1_000_000).unwrap(), @@ -1108,8 +1103,8 @@ async fn start_rpc_with_options( let store = TestStore::start().await; let block_producer_dir = new_tempdir(); TestStore::bootstrap(&block_producer_dir); - let block_producer_state = load_state(&block_producer_dir).await; - let store_state = Arc::clone(&store.state); + let (block_producer_state, ..) = State::for_tests(&block_producer_dir).await; + let state = Arc::clone(&store.state); // Start the rpc component. let rpc_listener = TcpListener::bind("127.0.0.1:0").await.expect("Failed to bind rpc"); @@ -1131,11 +1126,12 @@ async fn start_rpc_with_options( .connect_lazy::(); Rpc { listener: rpc_listener, - store: store_state, + state, mode: RpcMode::sequencer( block_producer, ValidatorClients::new(vec![validator]).unwrap(), ), + sync_writers: None, ntx_builder: None, grpc_options, network_tx_auth: None, diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index bd20adc7c5..35bdd7fe0b 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -19,6 +19,7 @@ doctest = false [dependencies] anyhow = { workspace = true } +arc-swap = { workspace = true } deadpool = { features = ["managed", "rt_tokio_1"], workspace = true } deadpool-diesel = { features = ["sqlite"], workspace = true } diesel = { features = ["numeric", "sqlite"], workspace = true } diff --git a/crates/store/src/account_state_forest/mod.rs b/crates/store/src/account_state_forest/mod.rs index 0f7c2775ed..34d3477c08 100644 --- a/crates/store/src/account_state_forest/mod.rs +++ b/crates/store/src/account_state_forest/mod.rs @@ -4,7 +4,7 @@ use std::num::NonZeroUsize; use miden_crypto::hash::rpo::Rpo256; #[cfg(feature = "rocksdb")] use miden_crypto::merkle::smt::ForestPersistentBackend; -use miden_crypto::merkle::smt::{Backend, ForestInMemoryBackend}; +use miden_crypto::merkle::smt::{Backend, BackendReader, ForestInMemoryBackend}; use miden_node_proto::domain::account::{ AccountStorageMapDetails, AccountVaultDetails, @@ -73,6 +73,9 @@ pub(crate) type AccountStateForestBackend = ForestPersistentBackend; #[cfg(not(feature = "rocksdb"))] pub(crate) type AccountStateForestBackend = ForestInMemoryBackend; +/// The read-only snapshot backend for the forest, used by in-memory state snapshots. +pub(crate) type AccountStateForestBackendReader = ::Reader; + const fn empty_smt_root() -> Word { *EmptySubtreeRoots::entry(SMT_DEPTH, 0) } @@ -89,7 +92,7 @@ pub enum AccountStorageMapResult { } /// Container for forest-related state that needs to be updated atomically. -pub(crate) struct AccountStateForest { +pub(crate) struct AccountStateForest { /// `LargeSmtForest` for efficient account storage reconstruction. Populated during block import /// with storage and vault SMTs. forest: LargeSmtForest, @@ -153,7 +156,7 @@ impl AccountStateForest { } } -impl AccountStateForest { +impl AccountStateForest { pub(crate) fn from_backend(backend: B) -> Result { Ok(Self { forest: LargeSmtForest::new(backend)?, @@ -735,6 +738,37 @@ impl AccountStateForest { ))) } + /// Retrieves the most recent vault SMT root for an account. If no vault root is found for the + /// account, returns an empty SMT root. + pub(crate) fn get_latest_vault_root(&self, account_id: AccountId) -> Word { + let lineage = Self::vault_lineage_id(account_id); + self.forest.latest_root(lineage).unwrap_or_else(empty_smt_root) + } + + /// Retrieves the most recent storage map SMT root for an account slot. + pub(crate) fn get_latest_storage_map_root( + &self, + account_id: AccountId, + slot_name: &StorageSlotName, + ) -> Word { + let lineage = Self::storage_lineage_id(account_id, slot_name); + self.forest.latest_root(lineage).unwrap_or_else(empty_smt_root) + } +} + +impl AccountStateForest { + /// Returns a read-only snapshot of this forest backed by a reader view of the backend. + /// + /// The reverse-key caches are shallow clones sharing the same underlying storage, so cache + /// entries added by the writer are visible to snapshot readers and vice versa. + pub(crate) fn reader(&self) -> Result, LargeSmtForestError> { + Ok(AccountStateForest { + forest: self.forest.reader()?, + storage_map_key_cache: self.storage_map_key_cache.clone(), + vault_key_cache: self.vault_key_cache.clone(), + }) + } + // PUBLIC INTERFACE // -------------------------------------------------------------------------------------------- @@ -857,23 +891,6 @@ impl AccountStateForest { self.apply_account_updates_without_pruning(block_num, account_updates) } - /// Retrieves the most recent vault SMT root for an account. If no vault root is found for the - /// account, returns an empty SMT root. - pub(crate) fn get_latest_vault_root(&self, account_id: AccountId) -> Word { - let lineage = Self::vault_lineage_id(account_id); - self.forest.latest_root(lineage).unwrap_or_else(empty_smt_root) - } - - /// Retrieves the most recent storage map SMT root for an account slot. - pub(crate) fn get_latest_storage_map_root( - &self, - account_id: AccountId, - slot_name: &StorageSlotName, - ) -> Word { - let lineage = Self::storage_lineage_id(account_id, slot_name); - self.forest.latest_root(lineage).unwrap_or_else(empty_smt_root) - } - // PRUNING // -------------------------------------------------------------------------------------------- diff --git a/crates/store/src/accounts/mod.rs b/crates/store/src/accounts/mod.rs index 4cc7fb930c..ee40da4389 100644 --- a/crates/store/src/accounts/mod.rs +++ b/crates/store/src/accounts/mod.rs @@ -16,6 +16,7 @@ use miden_protocol::crypto::merkle::smt::{ SMT_DEPTH, SmtLeaf, SmtStorage, + SmtStorageReader, }; use miden_protocol::crypto::merkle::{ EmptySubtreeRoots, @@ -118,7 +119,7 @@ impl HistoricalOverlay { /// reversion data (mutations that undo changes). Historical witnesses are reconstructed /// by starting from the latest state and applying reversion overlays backwards in time. #[derive(Debug)] -pub struct AccountTreeWithHistory { +pub struct AccountTreeWithHistory { /// The current block number (latest state). block_number: BlockNumber, /// The latest account tree state. @@ -127,7 +128,7 @@ pub struct AccountTreeWithHistory { overlays: BTreeMap, } -impl AccountTreeWithHistory { +impl AccountTreeWithHistory { /// Maximum number of historical blocks to maintain. pub const MAX_HISTORY: usize = 50; @@ -351,7 +352,9 @@ impl AccountTreeWithHistory { let path = SparseMerklePath::try_from(path).ok()?; Some((path, leaf)) } +} +impl AccountTreeWithHistory { // PUBLIC MUTATORS // -------------------------------------------------------------------------------------------- @@ -404,4 +407,14 @@ impl AccountTreeWithHistory { Ok(()) } + + /// Returns a read-only snapshot of this tree backed by a reader view of the storage. + pub fn reader(&self) -> AccountTreeWithHistory { + let latest = self.latest.reader().expect("snapshot creation should not fail"); + AccountTreeWithHistory { + block_number: self.block_number, + latest, + overlays: self.overlays.clone(), + } + } } diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 510e1b52da..2155eeaf9c 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -1,7 +1,7 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::mem::size_of; use std::num::NonZeroUsize; -use std::ops::{Deref, DerefMut, RangeInclusive}; +use std::ops::{Deref, DerefMut}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -36,7 +36,6 @@ use miden_protocol::note::{ }; use miden_protocol::transaction::TransactionHeader; use miden_protocol::utils::serde::Deserializable; -use tokio::sync::oneshot; use tracing::info; use crate::db::migrations::{migrate_database, verify_latest_schema}; @@ -55,6 +54,7 @@ use crate::db::models::queries::{ }; use crate::errors::{DatabaseError, NoteSyncError}; use crate::genesis::GenesisBlock; +use crate::state::{ScopedBlockNum, ScopedBlockRange}; use crate::{COMPONENT, LOG_TARGET}; const STORAGE_MAP_VALUE_PER_ROW_BYTES: usize = @@ -308,8 +308,9 @@ impl Db { &self, prefix_len: u32, nullifier_prefixes: Vec, - block_range: RangeInclusive, + block_range: ScopedBlockRange, ) -> Result<(Vec, BlockNumber)> { + let block_range = block_range.into_inner(); assert_eq!(prefix_len, 16, "Only 16-bit prefixes are supported"); self.transact("nullifieres by prefix", move |conn| { @@ -335,10 +336,13 @@ impl Db { )] pub async fn select_block_header_by_block_num( &self, - maybe_block_number: Option, + maybe_block_number: Option, ) -> Result> { self.transact("block headers by block number", move |conn| { - let val = queries::select_block_header_by_block_num(conn, maybe_block_number)?; + let val = queries::select_block_header_by_block_num( + conn, + maybe_block_number.map(|block_number| *block_number), + )?; Ok(val) }) .await @@ -353,10 +357,11 @@ impl Db { )] pub async fn select_block_header_and_signatures_by_block_num( &self, - block_number: BlockNumber, + block_number: ScopedBlockNum, ) -> Result> { self.transact("block headers and signatures by block number", move |conn| { - let val = queries::select_block_header_and_signatures_by_block_num(conn, block_number)?; + let val = + queries::select_block_header_and_signatures_by_block_num(conn, *block_number)?; Ok(val) }) .await @@ -370,10 +375,10 @@ impl Db { )] pub async fn select_block_headers( &self, - blocks: impl Iterator + Send + 'static, + blocks: impl Iterator + Send + 'static, ) -> Result> { self.transact("block headers from given block numbers", move |conn| { - let raw = queries::select_block_headers(conn, blocks)?; + let raw = queries::select_block_headers(conn, blocks.map(|block| *block))?; Ok(raw) }) .await @@ -497,10 +502,12 @@ impl Db { pub async fn select_account_header_with_storage_header_at_block( &self, account_id: AccountId, - block_num: BlockNumber, + block_num: ScopedBlockNum, ) -> Result> { self.transact("Get account header with storage header at block", move |conn| { - queries::select_account_header_with_storage_header_at_block(conn, account_id, block_num) + queries::select_account_header_with_storage_header_at_block( + conn, account_id, *block_num, + ) }) .await } @@ -512,9 +519,10 @@ impl Db { )] pub async fn get_note_sync_multi( &self, - block_range: RangeInclusive, + block_range: ScopedBlockRange, note_tags: Arc<[u32]>, ) -> Result, NoteSyncError> { + let block_range = block_range.into_inner(); self.transact("notes sync task", move |conn| { queries::get_note_sync_multi(conn, ¬e_tags, block_range, MAX_RESPONSE_PAYLOAD_BYTES) }) @@ -535,7 +543,8 @@ impl Db { .await } - /// Returns all note commitments from the DB that match the provided ones. + /// Returns all note commitments from the DB that match the provided ones and were committed at + /// or before `up_to_block`. #[miden_instrument( level = "debug", target = COMPONENT, @@ -544,14 +553,20 @@ impl Db { pub async fn select_existing_note_commitments( &self, note_commitments: Vec, + up_to_block: ScopedBlockNum, ) -> Result> { self.transact("note by commitment", move |conn| { - queries::select_existing_note_commitments(conn, note_commitments.as_slice()) + queries::select_existing_note_commitments( + conn, + note_commitments.as_slice(), + *up_to_block, + ) }) .await } - /// Loads inclusion proofs for notes matching the given note commitments. + /// Loads inclusion proofs for notes matching the given note commitments that were committed at + /// or before `up_to_block`. #[miden_instrument( level = "debug", target = COMPONENT, @@ -560,17 +575,23 @@ impl Db { pub async fn select_note_inclusion_proofs( &self, note_commitments: BTreeSet, + up_to_block: ScopedBlockNum, ) -> Result> { self.transact("block note inclusion proofs by commitment", move |conn| { - models::queries::select_note_inclusion_proofs(conn, ¬e_commitments) + models::queries::select_note_inclusion_proofs(conn, ¬e_commitments, *up_to_block) }) .await } /// Inserts the data of a new block into the DB. /// - /// `allow_acquire` and `acquire_done` are used to synchronize writes to the DB with writes to - /// the in-memory trees. Further details available on [`super::state::State::apply_block`]. + /// The transaction is committed when this method returns. Synchronization with the in-memory + /// trees is handled by the block writer task; see [`super::state::State::apply_block`]. + /// + /// Account history is pruned in the same transaction against `prune_tip`: the effective tip + /// for retention, which lags the actual tip while old snapshot generations are still pinned + /// by readers (SQLite reads have no point-in-time protection, unlike the `RocksDB`-backed + /// trees). /// /// Consumed note IDs omitted from transaction headers are resolved from /// `unresolved_note_nullifiers` on a best-effort basis. The returned mapping is used only for @@ -583,31 +604,18 @@ impl Db { )] pub(crate) async fn apply_block( &self, - allow_acquire: oneshot::Sender<()>, - acquire_done: oneshot::Receiver<()>, signed_block: SignedBlock, notes: Vec<(NoteRecord, Option)>, precomputed_public_states: PrecomputedPublicAccountStates, unresolved_note_nullifiers: Vec, + prune_tip: BlockNumber, ) -> Result> { self.transact("apply block", move |conn| { models::queries::apply_block(conn, &signed_block, ¬es, &precomputed_public_states)?; - - // XXX FIXME TODO free floating mutex MUST NOT exist it doesn't bind it properly to the - // data locked! - { - let _span = tracing::info_span!(target: COMPONENT, "acquire_write_lock").entered(); - if allow_acquire.send(()).is_err() { - tracing::warn!(target: COMPONENT, "failed to send notification for successful block application, potential deadlock"); - } - } - - models::queries::prune_history(conn, signed_block.header().block_num())?; + models::queries::prune_history(conn, prune_tip)?; let mut resolved_note_ids = BTreeMap::new(); - for chunk in - unresolved_note_nullifiers.chunks(QueryParamNoteCommitmentLimit::LIMIT) - { + for chunk in unresolved_note_nullifiers.chunks(QueryParamNoteCommitmentLimit::LIMIT) { match queries::select_note_ids_by_nullifier(conn, chunk) { Ok(note_ids) => resolved_note_ids.extend(note_ids), Err(err) => { @@ -622,10 +630,6 @@ impl Db { } } - let _span = - tracing::info_span!(target: COMPONENT, "acquire_done_lock").entered(); - acquire_done.blocking_recv()?; - Ok(resolved_note_ids) }) .await @@ -638,9 +642,10 @@ impl Db { pub(crate) async fn select_storage_map_sync_values( &self, account_id: AccountId, - block_range: RangeInclusive, + block_range: ScopedBlockRange, entries_limit: Option, ) -> Result { + let block_range = block_range.into_inner(); let entries_limit = entries_limit.unwrap_or_else(default_storage_map_entries_limit); self.transact("select storage map sync values", move |conn| { @@ -669,7 +674,7 @@ impl Db { &self, account_id: AccountId, slot_name: miden_protocol::account::StorageSlotName, - block_num: BlockNumber, + block_num: ScopedBlockNum, entries_limit: Option, ) -> Result { use miden_node_proto::domain::account::{AccountStorageMapDetails, StorageMapEntries}; @@ -684,7 +689,7 @@ impl Db { let mut page = self .select_storage_map_sync_values( account_id, - block_range_start..=block_num, + block_num.range_from(block_range_start), Some(entries_limit), ) .await?; @@ -699,7 +704,8 @@ impl Db { } loop { - if page.last_block_included == block_num || page.last_block_included < block_range_start + if page.last_block_included == *block_num + || page.last_block_included < block_range_start { break; } @@ -708,7 +714,7 @@ impl Db { page = self .select_storage_map_sync_values( account_id, - block_range_start..=block_num, + block_num.range_from(block_range_start), Some(entries_limit), ) .await?; @@ -721,7 +727,7 @@ impl Db { values.extend(page.values); } - if page.last_block_included != block_num { + if page.last_block_included != *block_num { return Ok(AccountStorageMapDetails::limit_exceeded(slot_name)); } @@ -758,10 +764,10 @@ impl Db { pub async fn select_account_vault_at_block( &self, account_id: AccountId, - block_num: BlockNumber, + block_num: ScopedBlockNum, ) -> Result, DatabaseError> { self.transact("select account vault at block", move |conn| { - queries::select_account_vault_at_block(conn, account_id, block_num) + queries::select_account_vault_at_block(conn, account_id, *block_num) }) .await } @@ -769,8 +775,9 @@ impl Db { pub async fn get_account_vault_sync( &self, account_id: AccountId, - block_range: RangeInclusive, + block_range: ScopedBlockRange, ) -> Result<(BlockNumber, Vec)> { + let block_range = block_range.into_inner(); self.transact("account vault sync", move |conn| { queries::select_account_vault_assets(conn, account_id, block_range) }) @@ -794,8 +801,9 @@ impl Db { pub async fn select_transactions_records( &self, account_ids: Vec, - block_range: RangeInclusive, + block_range: ScopedBlockRange, ) -> Result<(BlockNumber, Vec)> { + let block_range = block_range.into_inner(); self.transact("full transactions records", move |conn| { queries::select_transactions_records(conn, &account_ids, block_range) }) diff --git a/crates/store/src/db/models/queries/notes.rs b/crates/store/src/db/models/queries/notes.rs index f318193f07..f1fd1f2438 100644 --- a/crates/store/src/db/models/queries/notes.rs +++ b/crates/store/src/db/models/queries/notes.rs @@ -221,7 +221,8 @@ pub(crate) fn select_notes_by_id( Ok(records) } -/// Select the subset of note commitments that already exist in the notes table +/// Select the subset of note commitments that already exist in the notes table and were +/// committed at or before `up_to_block`. /// /// # Raw SQL /// @@ -229,11 +230,12 @@ pub(crate) fn select_notes_by_id( /// SELECT /// notes.note_commitment /// FROM notes -/// WHERE note_commitment IN (?1) +/// WHERE note_commitment IN (?1) AND committed_at <= ?2 /// ``` pub(crate) fn select_existing_note_commitments( conn: &mut SqliteConnection, note_commitments: &[Word], + up_to_block: BlockNumber, ) -> Result, DatabaseError> { QueryParamNoteCommitmentLimit::check(note_commitments.len())?; @@ -241,6 +243,7 @@ pub(crate) fn select_existing_note_commitments( let raw_commitments = SelectDsl::select(schema::notes::table, schema::notes::note_id) .filter(schema::notes::note_id.eq_any(¬e_commitments)) + .filter(schema::notes::committed_at.le(up_to_block.to_raw_sql())) .load::>(conn)?; let commitments = raw_commitments @@ -251,11 +254,13 @@ pub(crate) fn select_existing_note_commitments( Ok(commitments) } -/// Select note inclusion proofs matching the note commitments. +/// Select note inclusion proofs matching the note commitments, restricted to notes committed at +/// or before `up_to_block`. /// /// # Parameters /// * `note_ids`: Set of note IDs to query /// - Limit: 0 <= count <= 1000 +/// * `up_to_block`: Only notes committed at or before this block are returned /// /// # Returns /// @@ -274,13 +279,15 @@ pub(crate) fn select_existing_note_commitments( /// FROM /// notes /// WHERE -/// note_id IN (?1) +/// note_id IN (?1) AND +/// committed_at <= ?2 /// ORDER BY /// committed_at ASC /// ``` pub(crate) fn select_note_inclusion_proofs( conn: &mut SqliteConnection, note_commitments: &BTreeSet, + up_to_block: BlockNumber, ) -> Result, DatabaseError> { QueryParamNoteCommitmentLimit::check(note_commitments.len())?; @@ -297,6 +304,7 @@ pub(crate) fn select_note_inclusion_proofs( ), ) .filter(schema::notes::note_id.eq_any(note_commitments)) + .filter(schema::notes::committed_at.le(up_to_block.to_raw_sql())) .order_by(schema::notes::committed_at.asc()) .load::<(i64, Vec, i32, i32, Vec)>(conn)?; diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index 78b1ec5f4d..ec895449c6 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -82,6 +82,7 @@ use rand::RngExt; use tempfile::tempdir; use super::{AccountInfo, NoteRecord, NoteSyncRecord, NullifierInfo, TransactionRecord}; +use crate::ScopedBlockNum; use crate::account_state_forest::{ AccountStorageMapResult, HISTORICAL_BLOCK_RETENTION, @@ -1637,7 +1638,12 @@ async fn reconstruct_storage_map_from_db_pages_until_latest() { .unwrap(); let details = db - .reconstruct_storage_map_from_db(account_id, slot_name.clone(), block3, Some(1)) + .reconstruct_storage_map_from_db( + account_id, + slot_name.clone(), + ScopedBlockNum::new_unchecked(block3), + Some(1), + ) .await .unwrap(); @@ -1695,7 +1701,12 @@ async fn reconstruct_storage_map_from_db_returns_limit_exceeded_for_single_block // Use limit=1 so that 3 entries in a single block exceed the limit. block_range_start is block5 // (the first block with data), and the target is also block5. let details = db - .reconstruct_storage_map_from_db(account_id, slot_name.clone(), block5, Some(1)) + .reconstruct_storage_map_from_db( + account_id, + slot_name.clone(), + ScopedBlockNum::new_unchecked(block5), + Some(1), + ) .await .unwrap(); diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index 0969a58ad3..fb1f41843b 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -81,6 +81,8 @@ pub enum DatabaseError { Diesel(#[from] diesel::result::Error), #[error(transparent)] QueryParamLimit(#[from] QueryLimitError), + #[error(transparent)] + RangeBeyondTip(#[from] RangeBeyondTip), // OTHER ERRORS // --------------------------------------------------------------------------------------------- @@ -231,16 +233,16 @@ pub enum ApplyBlockError { // OTHER ERRORS // --------------------------------------------------------------------------------------------- - #[error("block applying was cancelled because of closed channel on database side")] + #[error("block applying was cancelled because the writer task dropped the result channel")] ClosedChannel(#[from] RecvError), - #[error("concurrent write detected")] - ConcurrentWrite, #[error("account state forest update preparation failed")] AccountStateForestPreparation(#[source] AccountStateForestUpdateError), #[error("database doesn't have any block header data")] DbBlockHeaderEmpty, #[error("database update failed: {0}")] DbUpdateTaskFailed(String), + #[error("failed to send block to the writer task: {0}")] + WriterTaskSendFailed(String), } #[derive(Error, Debug)] @@ -251,6 +253,14 @@ pub enum ApplyBlockWithProvingInputsError { ApplyBlock(#[source] ApplyBlockError), } +/// A requested block range extends beyond the chain tip of the state view serving the request. +#[derive(Error, Debug)] +#[error("block_to ({block_to}) is greater than chain tip ({chain_tip})")] +pub struct RangeBeyondTip { + pub chain_tip: BlockNumber, + pub block_to: BlockNumber, +} + #[derive(Error, Debug)] pub enum GetBlockHeaderError { #[error("database error")] @@ -282,6 +292,8 @@ pub enum StateSyncError { EmptyBlockHeadersTable, #[error("failed to build MMR delta")] FailedToBuildMmrDelta(#[from] MmrError), + #[error(transparent)] + RangeBeyondTip(#[from] RangeBeyondTip), } impl From for StateSyncError { @@ -302,11 +314,8 @@ pub enum NoteSyncError { MmrError(#[from] MmrError), #[error("invalid block range")] InvalidBlockRange(#[from] InvalidBlockRange), - #[error("block_to ({block_to}) is greater than chain tip ({chain_tip})")] - FutureBlock { - chain_tip: BlockNumber, - block_to: BlockNumber, - }, + #[error(transparent)] + RangeBeyondTip(#[from] RangeBeyondTip), #[error("malformed note tags")] DeserializationFailed(#[from] ConversionError), } diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 416e087367..ec40d3fe3b 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -35,10 +35,20 @@ pub use errors::{ GetBlockHeaderError, GetBlockInputsError, NoteSyncError, + RangeBeyondTip, StateSyncError, }; pub use genesis::GenesisState; -pub use state::State; +pub use state::{ + BlockWriter, + LoadedState, + ProofWriter, + ScopedBlockNum, + ScopedBlockRange, + State, + StateView, + WriterTask, +}; /// Returns the store crate version. pub fn version() -> &'static str { diff --git a/crates/store/src/state/apply_block.rs b/crates/store/src/state/apply_block.rs deleted file mode 100644 index f0a9f2a1e2..0000000000 --- a/crates/store/src/state/apply_block.rs +++ /dev/null @@ -1,429 +0,0 @@ -use std::fmt::Display; -use std::sync::Arc; - -use miden_node_proto::domain::proof_request::BlockProofRequest; -use miden_node_utils::ErrorReport; -use miden_node_utils::tracing::{miden_instrument, miden_span_record}; -use miden_protocol::Word; -use miden_protocol::account::AccountUpdateDetails; -use miden_protocol::batch::OrderedBatches; -use miden_protocol::block::account_tree::AccountMutationSet; -use miden_protocol::block::nullifier_tree::NullifierMutationSet; -use miden_protocol::block::{BlockBody, BlockHeader, BlockInputs, BlockNumber, SignedBlock}; -use miden_protocol::note::{NoteDetails, Nullifier}; -use miden_protocol::transaction::OutputNote; -use miden_protocol::utils::serde::Serializable; -use tokio::sync::oneshot; -use tracing::{Instrument, info_span}; - -use crate::db::NoteRecord; -use crate::errors::{ApplyBlockError, ApplyBlockWithProvingInputsError, InvalidBlockError}; -use crate::state::block_lifecycle::{BlockLifecycle, lifecycle_events_enabled}; -use crate::state::{BlockNotification, InnerState, State}; -use crate::{COMPONENT, HistoricalError, LOG_TARGET}; - -impl State { - /// Saves proving inputs for a signed block and applies it to the state. - /// - /// Used by the in-process block producer after it has built and signed a block. - #[miden_instrument( - target = COMPONENT, - err, - )] - pub async fn apply_block_with_proving_inputs( - &self, - ordered_batches: OrderedBatches, - block_inputs: BlockInputs, - signed_block: SignedBlock, - ) -> Result<(), ApplyBlockWithProvingInputsError> { - let block_header = signed_block.header().clone(); - let block_num = block_header.block_num(); - - let proving_inputs = BlockProofRequest { - tx_batches: ordered_batches, - block_header, - block_inputs, - }; - - self.save_proving_inputs(block_num, &proving_inputs) - .await - .map_err(ApplyBlockWithProvingInputsError::SaveProvingInputs)?; - - self.apply_block(signed_block) - .await - .map_err(ApplyBlockWithProvingInputsError::ApplyBlock) - } - - /// Apply changes of a new block to the DB and in-memory data structures. - /// - /// ## Note on state consistency - /// - /// The server contains in-memory representations of the existing trees, the in-memory - /// representation must be kept consistent with the committed data, this is necessary so to - /// provide consistent results for all endpoints. In order to achieve consistency, the - /// following steps are used: - /// - /// - the request data is validated, prior to starting any modifications. - /// - block is being saved into the store in parallel with updating the DB, but before - /// committing. This block is considered as candidate and not yet available for reading - /// because the latest block pointer is not updated yet. - /// - a transaction is open in the DB and the writes are started. - /// - while the transaction is not committed, concurrent reads are allowed, both the DB and the - /// in-memory representations, which are consistent at this stage. - /// - prior to committing the changes to the DB, exclusive locks to the canonical in-memory - /// state and account-state forest are acquired, preventing readers from observing them at - /// different block heights. - /// - the DB transaction is committed, and requests that read only from the DB can proceed to - /// use the fresh data. - /// - the account-state forest and canonical in-memory structures are updated, including the - /// latest block pointer, and both locks are released. - /// - if any persistent tree update fails after the DB commit, the process aborts. The durable - /// database remains authoritative, and divergent tree storage must be rebuilt before the node - /// resumes normal processing. - // TODO: This span is logged in a root span, we should connect it to the parent span. - #[miden_instrument( - target = COMPONENT, - err, - )] - pub async fn apply_block(&self, signed_block: SignedBlock) -> Result<(), ApplyBlockError> { - let _lock = self.writer.try_lock().map_err(|_| ApplyBlockError::ConcurrentWrite)?; - - let header = signed_block.header(); - let body = signed_block.body(); - - let block_num = header.block_num(); - let block_commitment = header.commitment(); - let num_transactions = body.transactions().as_slice().len(); - - miden_span_record!( - block.number = %block_num, - block.commitment = %block_commitment, - block.transactions.count = num_transactions, - ); - - self.validate_block_header(header, body).await?; - - let block_lifecycle = - lifecycle_events_enabled().then(|| BlockLifecycle::from_block_body(block_num, body)); - let unresolved_note_nullifiers = block_lifecycle - .as_ref() - .map_or_else(Vec::new, BlockLifecycle::unresolved_note_nullifiers); - - // Save the block to the block store. In a case of a rolled-back DB transaction, the - // in-memory state will be unchanged, but the file might still be written. Such blocks - // should be considered candidates, not finalized blocks. - let signed_block_bytes = signed_block.to_bytes(); - // Clone before moving into the block-save task so we can cache for replicas at commit. - let cache_bytes = signed_block_bytes.clone(); - let store = Arc::clone(&self.block_store); - let block_save_task = tokio::spawn( - async move { store.save_block(block_num, &signed_block_bytes).await }.in_current_span(), - ); - - let ( - nullifier_tree_old_root, - nullifier_tree_update, - account_tree_old_root, - account_tree_update, - ) = self.compute_tree_mutations(header, body).await?; - - let notes = Self::build_note_records(header, body)?; - - // Signals the transaction is ready to be committed, and the write lock can be acquired. - let (allow_acquire, acquired_allowed) = oneshot::channel::<()>(); - // Signals the write lock has been acquired, and the transaction can be committed. - let (inform_acquire_done, acquire_done) = oneshot::channel::<()>(); - - // Extract public account updates with patches before block is moved into async task. - // Private accounts are filtered out since they don't expose their state changes. - let account_patches = - Vec::from_iter(body.updated_accounts().iter().filter_map( - |update| match update.details() { - AccountUpdateDetails::Public(patch) => Some(patch.clone()), - AccountUpdateDetails::Private => None, - }, - )); - let account_forest_update = self.with_forest_read_blocking(|forest| { - forest - .compute_block_update_mutations(block_num, account_patches) - .map_err(ApplyBlockError::AccountStateForestPreparation) - })?; - let precomputed_public_states = account_forest_update.account_states.clone(); - - // The DB and in-memory state updates need to be synchronized and are partially overlapping. - // Namely, the DB transaction only proceeds after this task acquires the in-memory write - // lock. This requires the DB update to run concurrently, so a new task is spawned. - let db = Arc::clone(&self.db); - let db_update_task = tokio::spawn( - async move { - db.apply_block( - allow_acquire, - acquire_done, - signed_block, - notes, - precomputed_public_states, - unresolved_note_nullifiers, - ) - .await - } - .in_current_span(), - ); - - // Wait for the message from the DB update task, that we ready to commit the DB transaction. - acquired_allowed - .instrument(info_span!(target: COMPONENT, "await_db_readiness")) - .await - .map_err(ApplyBlockError::ClosedChannel)?; - - // Awaiting the block saving task to complete without errors. - block_save_task.await??; - - let resolved_note_ids = self.with_inner_and_forest_write_blocking(|inner, forest| { - // We need to check that neither the nullifier tree nor the account tree have changed - // while we were waiting for the DB preparation task to complete. If either of them did - // change, we do not proceed with in-memory and database updates, since it may lead to - // an inconsistent state. - if inner.nullifier_tree.root() != nullifier_tree_old_root - || inner.account_tree.root_latest() != account_tree_old_root - { - return Err(ApplyBlockError::ConcurrentWrite); - } - - // Notify the DB update task that the write lock has been acquired, so it can commit the - // DB transaction. - inform_acquire_done - .send(()) - .map_err(|_| ApplyBlockError::DbUpdateTaskFailed("Receiver was dropped".into()))?; - - // TODO: shutdown #91 Await for successful commit of the DB transaction. If the commit - // fails, we mustn't change in-memory state, so we return a block applying error and - // don't proceed with in-memory updates. - let resolved_note_ids = tokio::runtime::Handle::current() - .block_on(db_update_task)? - .map_err(|err| ApplyBlockError::DbUpdateTaskFailed(err.as_report()))?; - - // The DB is now committed. Keep both write locks held while advancing the forest and - // canonical in-memory state so readers cannot observe different block heights. - let InnerState { nullifier_tree, blockchain, account_tree } = inner; - forest - .apply_precomputed_block_update(block_num, account_forest_update) - .unwrap_or_else(|error| { - Self::abort_after_post_commit_failure("account-state forest", &error) - }); - nullifier_tree.apply_mutations(nullifier_tree_update).unwrap_or_else(|error| { - Self::abort_after_post_commit_failure("nullifier tree", &error) - }); - account_tree.apply_mutations(account_tree_update).unwrap_or_else(|error| { - Self::abort_after_post_commit_failure("account tree", &error) - }); - blockchain.push(block_commitment); - - Ok(resolved_note_ids) - })?; - - // Push to cache and notify replica subscribers. - self.block_cache - .push(block_num, BlockNotification::new(block_num, cache_bytes)) - .expect("block cache receives sequential block numbers"); - let _ = self.committed_tip_tx.send(block_num); - - if let Some(block_lifecycle) = block_lifecycle { - block_lifecycle.emit(&resolved_note_ids); - } - tracing::debug!(target: LOG_TARGET, "Block applied"); - - Ok(()) - } - - /// Terminates after a persistent state failure that occurred after the canonical DB commit. - /// - /// Returning would expose components at different block heights. Tests panic so the fatal path - /// can be asserted without terminating the test process; production aborts immediately. - fn abort_after_post_commit_failure(component: &str, error: &impl Display) -> ! { - tracing::error!( - target: LOG_TARGET, - component, - error = %error, - "persistent state update failed after database commit; aborting" - ); - - #[cfg(test)] - panic!("{component} update failed after database commit: {error}"); - - #[cfg(not(test))] - std::process::abort(); - } - - /// Saves the proving inputs for the given block to the block store. - pub async fn save_proving_inputs( - &self, - block_num: BlockNumber, - proving_inputs: &BlockProofRequest, - ) -> std::io::Result<()> { - self.block_store - .save_proving_inputs(block_num, &proving_inputs.to_bytes()) - .await - } - - /// Validates that the block header is consistent with the block body and the current state. - #[miden_instrument( - target = COMPONENT, - err, - )] - async fn validate_block_header( - &self, - header: &BlockHeader, - body: &BlockBody, - ) -> Result<(), ApplyBlockError> { - // Validate that header and body match. - let tx_commitment = body.transactions().commitment(); - if header.tx_commitment() != tx_commitment { - return Err(InvalidBlockError::InvalidBlockTxCommitment { - expected: tx_commitment, - actual: header.tx_commitment(), - } - .into()); - } - - let block_num = header.block_num(); - - // Validate that the applied block is the next block in sequence. - let prev_block = self - .db - .select_block_header_by_block_num(None) - .await? - .ok_or(ApplyBlockError::DbBlockHeaderEmpty)?; - let expected_block_num = prev_block.block_num().child(); - if block_num != expected_block_num { - return Err(InvalidBlockError::NewBlockInvalidBlockNum { - expected: expected_block_num, - submitted: block_num, - } - .into()); - } - if header.prev_block_commitment() != prev_block.commitment() { - return Err(InvalidBlockError::NewBlockInvalidPrevCommitment.into()); - } - - Ok(()) - } - - /// Computes nullifier and account tree mutations, validating roots against the block header. - #[miden_instrument( - target = COMPONENT, - err, - )] - async fn compute_tree_mutations( - &self, - header: &BlockHeader, - body: &BlockBody, - ) -> Result<(Word, NullifierMutationSet, Word, AccountMutationSet), ApplyBlockError> { - self.with_inner_read_blocking(|inner| { - let block_num = header.block_num(); - - // nullifiers can be produced only once - let duplicate_nullifiers: Vec<_> = body - .created_nullifiers() - .iter() - .filter(|&nullifier| inner.nullifier_tree.get_block_num(nullifier).is_some()) - .copied() - .collect(); - if !duplicate_nullifiers.is_empty() { - return Err(InvalidBlockError::DuplicatedNullifiers(duplicate_nullifiers).into()); - } - - // new_block.chain_root must be equal to the chain MMR root prior to the update - let peaks = inner.blockchain.peaks(); - if peaks.hash_peaks() != header.chain_commitment() { - return Err(InvalidBlockError::NewBlockInvalidChainCommitment.into()); - } - - // compute update for nullifier tree - let nullifier_tree_update = inner - .nullifier_tree - .compute_mutations( - body.created_nullifiers().iter().map(|nullifier| (*nullifier, block_num)), - ) - .map_err(InvalidBlockError::NewBlockNullifierAlreadySpent)?; - - if nullifier_tree_update.as_mutation_set().root() != header.nullifier_root() { - return Err(InvalidBlockError::NewBlockInvalidNullifierRoot.into()); - } - - // compute update for account tree - let account_tree_update = inner - .account_tree - .compute_mutations( - body.updated_accounts() - .iter() - .map(|update| (update.account_id(), update.final_state_commitment())), - ) - .map_err(|e| match e { - HistoricalError::AccountTreeError(err) => { - InvalidBlockError::NewBlockDuplicateAccountIdPrefix(err) - }, - HistoricalError::MerkleError(_) => { - panic!("Unexpected MerkleError during account tree mutation computation") - }, - })?; - - if account_tree_update.as_mutation_set().root() != header.account_root() { - return Err(InvalidBlockError::NewBlockInvalidAccountRoot.into()); - } - - Ok(( - inner.nullifier_tree.root(), - nullifier_tree_update, - inner.account_tree.root_latest(), - account_tree_update, - )) - }) - } - - /// Builds note records with inclusion proofs from the block body. - #[miden_instrument( - target = COMPONENT, - err, - )] - fn build_note_records( - header: &BlockHeader, - body: &BlockBody, - ) -> Result)>, ApplyBlockError> { - let block_num = header.block_num(); - - let note_tree = body.compute_block_note_tree(); - if note_tree.root() != header.note_root() { - return Err(InvalidBlockError::NewBlockInvalidNoteRoot.into()); - } - - let notes = body - .output_notes() - .map(|(note_index, note)| { - let (details, attachments, nullifier) = match note { - OutputNote::Public(public) => ( - Some(NoteDetails::from(public.as_note())), - public.as_note().attachments().clone(), - Some(public.as_note().nullifier()), - ), - OutputNote::Private(private) => (None, private.attachments().clone(), None), - }; - - let inclusion_path = note_tree.open(note_index); - - let note_record = NoteRecord { - block_num, - note_index, - note_id: note.id().as_word(), - metadata: *note.metadata(), - details, - attachments, - inclusion_path, - }; - - Ok((note_record, nullifier)) - }) - .collect::, InvalidBlockError>>()?; - - Ok(notes) - } -} diff --git a/crates/store/src/state/lifecycle.rs b/crates/store/src/state/lifecycle.rs new file mode 100644 index 0000000000..f12d7b2986 --- /dev/null +++ b/crates/store/src/state/lifecycle.rs @@ -0,0 +1,268 @@ +//! Store lifecycle: loading the state, starting its write worker, and stopping the store. + +use std::num::NonZeroUsize; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::AtomicUsize; + +use arc_swap::ArcSwap; +use miden_node_utils::ErrorReport; +use miden_node_utils::clap::StorageOptions; +use miden_node_utils::shutdown::CancellationToken; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::block::BlockNumber; +use tokio::sync::{mpsc, watch}; + +use crate::account_state_forest::AccountStateForestBackend; +use crate::accounts::AccountTreeWithHistory; +use crate::blocks::BlockStore; +use crate::db::Db; +use crate::errors::StateInitializationError; +use crate::proven_tip::ProvenTipWriter; +use crate::state::loader::{ + ACCOUNT_STATE_FOREST_STORAGE_DIR, + ACCOUNT_TREE_STORAGE_DIR, + AccountForestLoader, + NULLIFIER_TREE_STORAGE_DIR, + TreeStorage, + TreeStorageLoader, + load_mmr, + verify_account_state_forest_consistency, + verify_tree_consistency, +}; +use crate::state::writer::{WriteRequest, WriteWorker, WriterTask}; +use crate::state::{ + BlockCache, + BlockWriter, + ProofCache, + ProofWriter, + SnapshotGuard, + State, + StateSnapshot, +}; +use crate::{COMPONENT, DataDirectory, DatabaseOptions}; + +/// Number of recent committed blocks held in the in-memory cache for replica subscriptions. +const BLOCK_CACHE_CAPACITY: NonZeroUsize = NonZeroUsize::new(512).unwrap(); + +/// Number of recent block proofs held in the in-memory cache for replica subscriptions. +const PROOF_CACHE_CAPACITY: NonZeroUsize = NonZeroUsize::new(512).unwrap(); + +// LOADED STATE +// ================================================================================================ + +/// A loaded store state whose write worker has not been started yet. +/// +/// Returned by [`State::load`]. [`Self::start`] spawns the writer and yields the read-only +/// [`State`] together with the write capabilities; since this is the only way to obtain a +/// [`BlockWriter`], [`BlockWriter::apply_block`] can always make progress. +#[must_use = "call `start` to spawn the write worker and obtain the state"] +pub struct LoadedState { + state: State, + writer: WriteWorker, + write_tx: mpsc::Sender, +} + +impl LoadedState { + /// Spawns the write worker onto the current runtime and returns the read-only state together + /// with the write capabilities and the writer task's handle. + /// + /// [`Arc`] is the read-only view shared with every component that queries or + /// subscribes; it exposes no mutating methods. [`BlockWriter`] and [`ProofWriter`] are the + /// only handles able to mutate the store — hand them to the single task driving each write + /// path (block production or sync, and proof scheduling or sync respectively). The + /// capabilities expose no read access: tasks that both read and write receive the + /// [`Arc`] alongside their capability. + /// + /// The writer exits once `shutdown` is cancelled or the [`BlockWriter`] (holding the only + /// request sender) is dropped — an in-flight block write always completes first. Awaiting the + /// returned handle after either event guarantees the writer has released the tree storage it + /// owns; a join error carries a writer panic. + /// + /// Callers without a token to cancel should pass [`CancellationToken::new`] and stop the store + /// via [`BlockWriter::stop`] instead of dropping and joining by hand. + pub fn start( + self, + shutdown: CancellationToken, + ) -> (Arc, BlockWriter, ProofWriter, WriterTask) { + let writer_task = tokio::spawn(self.writer.run(shutdown)); + let state = Arc::new(self.state); + let block_writer = BlockWriter { + block_store: Arc::clone(&state.block_store), + write_tx: self.write_tx, + }; + let proof_writer = ProofWriter { state: Arc::clone(&state) }; + (state, block_writer, proof_writer, WriterTask(writer_task)) + } +} + +// LOAD +// ================================================================================================ + +impl State { + /// Loads the state from the data directory. + /// + /// The loaded state owns all store data structures and exposes subscription methods for + /// sequencer and replica tasks. Call [`LoadedState::start`] on the result to spawn the block + /// writer and obtain the usable [`State`]. + #[miden_instrument( + target = COMPONENT, + )] + pub async fn load( + data_path: &Path, + storage_options: StorageOptions, + ) -> Result { + Self::load_with_database_options(data_path, storage_options, DatabaseOptions::default()) + .await + } + + /// Loads the state from the data directory using explicit database options. + /// + /// The loaded state owns all store data structures and exposes subscription methods for + /// sequencer and replica tasks. Call [`LoadedState::start`] on the result to spawn the block + /// writer and obtain the usable [`State`]. + #[miden_instrument( + target = COMPONENT, + )] + pub async fn load_with_database_options( + data_path: &Path, + storage_options: StorageOptions, + database_options: DatabaseOptions, + ) -> Result { + let data_directory = DataDirectory::load(data_path.to_path_buf()) + .map_err(StateInitializationError::DataDirectoryLoadError)?; + + let block_store = Arc::new( + BlockStore::load(data_directory.block_store_dir()) + .map_err(StateInitializationError::BlockStoreLoadError)?, + ); + + let database_filepath = data_directory.database_path(); + let mut db = Db::load_with_pool_size( + database_filepath.clone(), + database_options.connection_pool_size, + ) + .await + .map_err(StateInitializationError::DatabaseLoadError)?; + + let blockchain = load_mmr(&mut db).await?; + let latest_block_num = blockchain.chain_tip().unwrap_or(BlockNumber::GENESIS); + + #[cfg(feature = "rocksdb")] + let (account_storage_config, nullifier_storage_config, forest_storage_config) = ( + storage_options.account_tree.into(), + storage_options.nullifier_tree.into(), + storage_options.account_state_forest.into(), + ); + #[cfg(not(feature = "rocksdb"))] + let (account_storage_config, nullifier_storage_config, forest_storage_config) = { + let _ = &storage_options; + ((), (), ()) + }; + let account_storage = + TreeStorage::create(data_path, &account_storage_config, ACCOUNT_TREE_STORAGE_DIR)?; + let account_tree = account_storage.load_account_tree(&mut db).await?; + + let nullifier_storage = + TreeStorage::create(data_path, &nullifier_storage_config, NULLIFIER_TREE_STORAGE_DIR)?; + let nullifier_tree = nullifier_storage.load_nullifier_tree(&mut db).await?; + + // Verify that tree roots match the expected roots from the database. This catches any + // divergence between persistent storage and the database caused by corruption or incomplete + // shutdown. + verify_tree_consistency(account_tree.root(), nullifier_tree.root(), &mut db).await?; + + let account_tree = AccountTreeWithHistory::new(account_tree, latest_block_num); + + let forest_backend = AccountStateForestBackend::create( + data_path, + &forest_storage_config, + ACCOUNT_STATE_FOREST_STORAGE_DIR, + )?; + let forest = forest_backend.load_account_state_forest(&mut db, latest_block_num).await?; + verify_account_state_forest_consistency(&forest, &mut db).await?; + + let db = Arc::new(db); + + // Initialize the proven tip from the block store. + let proven_tip_init = block_store + .load_proven_tip() + .map_err(StateInitializationError::ProvenTipLoadError)?; + let (proven_tip, _rx) = ProvenTipWriter::new(proven_tip_init); + + // Committed-tip watch: fires after each successful apply_block. + let (committed_tip_tx, _rx) = watch::channel(latest_block_num); + let committed_tip_tx = Arc::new(committed_tip_tx); + + let block_cache = BlockCache::new(BLOCK_CACHE_CAPACITY); + let proof_cache = ProofCache::new(PROOF_CACHE_CAPACITY); + + // Shared counter of live snapshot generations, for observability. + let snapshots_live = Arc::new(AtomicUsize::new(0)); + + // Create the initial snapshot from reader views of the just-loaded trees. + let initial_snapshot = Arc::new(StateSnapshot::new( + nullifier_tree + .reader() + .map_err(|e| StateInitializationError::NullifierTreeIoError(e.as_report()))?, + blockchain.clone(), + account_tree.reader(), + forest + .reader() + .map_err(|e| StateInitializationError::AccountStateForestIoError(e.as_report()))?, + SnapshotGuard::new(Arc::clone(&snapshots_live), latest_block_num), + )); + let latest_snapshot = Arc::new(ArcSwap::from(initial_snapshot)); + + // Assemble the write worker. It owns the writable trees and processes write requests + // serially, publishing a new snapshot after each committed block. The caller runs it; it + // exits when the shutdown token is cancelled or the `BlockWriter` (holding the only request + // sender) is dropped. + let (write_tx, write_rx) = mpsc::channel(1); + let block_writer = WriteWorker::new( + Arc::clone(&db), + Arc::clone(&block_store), + Arc::clone(&latest_snapshot), + Arc::clone(&committed_tip_tx), + block_cache.clone(), + write_rx, + nullifier_tree, + account_tree, + blockchain, + forest, + snapshots_live, + ); + let state = Self { + data_directory: data_path.to_path_buf(), + db, + block_store, + latest_snapshot, + proven_tip, + committed_tip_tx, + block_cache, + proof_cache, + }; + + Ok(LoadedState { state, writer: block_writer, write_tx }) + } + + /// Loads the state with default options and starts its write worker, detaching the worker + /// task. + /// + /// Test-only helper for tests in sibling crates that don't manage the writer's lifecycle: + /// the detached writer exits once the returned [`BlockWriter`] is dropped. Hidden from public + /// docs and not part of the stable API. + /// + /// # Panics + /// + /// Panics if the state fails to load. + #[doc(hidden)] + pub async fn for_tests(data_path: &Path) -> (Arc, BlockWriter, ProofWriter) { + let (state, block_writer, proof_writer, _writer_task) = + Self::load(data_path, StorageOptions::default()) + .await + .expect("state should load") + .start(CancellationToken::new()); + (state, block_writer, proof_writer) + } +} diff --git a/crates/store/src/state/loader.rs b/crates/store/src/state/loader.rs index 10f2c5d696..a2cdd45de4 100644 --- a/crates/store/src/state/loader.rs +++ b/crates/store/src/state/loader.rs @@ -76,6 +76,9 @@ pub type TreeStorage = RocksDbStorage; #[cfg(not(feature = "rocksdb"))] pub type TreeStorage = MemoryStorage; +/// The read-only snapshot storage backend for trees, used by in-memory state snapshots. +pub type TreeStorageReader = ::Reader; + // ERROR CONVERSION // ================================================================================================ diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index e80d2993ae..3a519e36a9 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -3,125 +3,43 @@ //! The [State] provides data access and modifications methods, its main purpose is to ensure that //! data is atomically written, and that reads are consistent. -use std::collections::{BTreeMap, BTreeSet, HashSet}; -use std::num::NonZeroUsize; -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use miden_node_proto::domain::batch::BatchInputs; -use miden_node_utils::clap::StorageOptions; -use miden_node_utils::formatting::format_array; -use miden_node_utils::tracing::miden_instrument; -use miden_protocol::Word; -use miden_protocol::account::AccountId; -use miden_protocol::block::account_tree::AccountWitness; -use miden_protocol::block::nullifier_tree::{NullifierTree, NullifierWitness}; -use miden_protocol::block::{BlockHeader, BlockInputs, BlockNumber, Blockchain}; -use miden_protocol::crypto::merkle::mmr::{MmrProof, PartialMmr}; -use miden_protocol::crypto::merkle::smt::{LargeSmt, SmtStorage}; -use miden_protocol::note::{NoteId, NoteScript, Nullifier}; -use miden_protocol::transaction::PartialBlockchain; -use tokio::sync::{Mutex, RwLock, watch}; -use tracing::{Instrument, Span}; - -use crate::account_state_forest::{AccountStateForest, AccountStateForestBackend}; -use crate::accounts::AccountTreeWithHistory; -use crate::blocks::BlockStore; -use crate::db::{Db, NoteRecord, NullifierInfo}; -use crate::errors::{ - DatabaseError, - GetBatchInputsError, - GetBlockHeaderError, - GetBlockInputsError, - StateInitializationError, -}; -use crate::proven_tip::ProvenTipWriter; -use crate::{COMPONENT, DataDirectory, DatabaseOptions}; - -/// Number of recent committed blocks held in the in-memory cache for replica subscriptions. -const BLOCK_CACHE_CAPACITY: NonZeroUsize = NonZeroUsize::new(512).unwrap(); - -/// Number of recent block proofs held in the in-memory cache for replica subscriptions. -const PROOF_CACHE_CAPACITY: NonZeroUsize = NonZeroUsize::new(512).unwrap(); - -mod loader; -use loader::{ - ACCOUNT_STATE_FOREST_STORAGE_DIR, - ACCOUNT_TREE_STORAGE_DIR, - AccountForestLoader, - NULLIFIER_TREE_STORAGE_DIR, - TreeStorage, - TreeStorageLoader, - load_mmr, - verify_account_state_forest_consistency, - verify_tree_consistency, -}; - -mod replica; -pub use replica::{BlockCache, BlockNotification, ProofCache, ProofNotification}; - -mod account; - -mod apply_block; -mod apply_proof; mod block_lifecycle; mod bootstrap; mod disk_monitor; -mod sync_state; - -// FINALITY -// ================================================================================================ - -/// The finality level for chain tip queries. -#[derive(Debug, Clone, Copy)] -pub enum Finality { - /// The latest committed (but not necessarily proven) block. - Committed, - /// The latest block that has been proven in an unbroken sequence from genesis. - Proven, -} - -// STRUCTURES -// ================================================================================================ - -#[derive(Debug, Default)] -pub struct TransactionInputs { - pub account_commitment: Word, - pub nullifiers: Vec, - pub found_unauthenticated_notes: HashSet, - pub new_account_id_prefix_is_unique: Option, -} +mod lifecycle; +mod loader; +mod replica; +mod tip; +mod view; +mod writer; -type BlockInputWitnesses = ( - BlockNumber, - BTreeMap, - BTreeMap, - PartialMmr, -); +use std::path::PathBuf; +use std::sync::Arc; -/// Container for state that needs to be updated atomically. -struct InnerState -where - S: SmtStorage, -{ - nullifier_tree: NullifierTree>, - blockchain: Blockchain, - account_tree: AccountTreeWithHistory, -} +use arc_swap::ArcSwap; +pub use lifecycle::LoadedState; +use miden_protocol::block::BlockNumber; +pub use replica::{BlockCache, BlockNotification, ProofCache, ProofNotification}; +use tokio::sync::watch; +pub use view::{ScopedBlockNum, ScopedBlockRange, StateView, TransactionInputs}; +use view::{SnapshotGuard, StateSnapshot}; +pub use writer::{BlockWriter, ProofWriter, WriterTask}; -impl InnerState { - /// Returns the latest block number. - fn latest_block_num(&self) -> BlockNumber { - self.blockchain - .chain_tip() - .expect("chain should always have at least the genesis block") - } -} +use crate::blocks::BlockStore; +use crate::db::Db; +use crate::proven_tip::ProvenTipWriter; // CHAIN STATE // ================================================================================================ -/// The rollup state. +/// The rollup state, read-only. +/// +/// Mutations go through the [`BlockWriter`] and [`ProofWriter`] capabilities returned by +/// [`LoadedState::start`]; every holder of this type can only query and subscribe. +/// +/// All tree and database reads go through a request-scoped [`StateView`] obtained from +/// [`State::view`]; this type itself only exposes chain-tip queries, subscriptions, and +/// block-store access. pub struct State { /// Root directory containing the store's on-disk data. data_directory: PathBuf, @@ -133,24 +51,19 @@ pub struct State { /// The block store which stores full block contents for all blocks. block_store: Arc, - /// Read-write lock used to prevent writing to a structure while it is being used. + /// Atomically swappable pointer to the latest in-memory state snapshot. /// - /// The lock is writer-preferring, meaning the writer won't be starved. - inner: RwLock>, - - /// Forest-related state `(SmtForest, storage_map_roots, vault_roots)` with its own lock. - forest: RwLock>, - - /// To allow readers to access the tree data while an update in being performed, and prevent - /// TOCTOU issues, there must be no concurrent writers. This locks to serialize the writers. - writer: Mutex<()>, + /// Readers load the snapshot wait-free via [`ArcSwap::load_full`]; the + /// [`WriteWorker`](writer::WriteWorker) task atomically replaces the pointer after each + /// committed block. Readers holding an old snapshot are unaffected by the swap. + latest_snapshot: Arc>, /// The latest proven-in-sequence block number, updated by the proof scheduler or `apply_proof`. proven_tip: ProvenTipWriter, /// Watch sender fired after each block is committed. Replicas subscribe via /// `subscribe_committed_tip()` to be woken when new blocks arrive. - committed_tip_tx: watch::Sender, + committed_tip_tx: Arc>, /// FIFO cache of recent committed blocks for replica subscriptions. When a subscriber needs a /// block that has been evicted, it falls back to loading from the block store. @@ -160,608 +73,3 @@ pub struct State { /// that has been evicted, it falls back to loading from the block store. pub(crate) proof_cache: ProofCache, } - -impl State { - // CONSTRUCTOR - // -------------------------------------------------------------------------------------------- - - /// Loads the state from the data directory. - /// - /// The loaded state owns all store data structures and exposes subscription methods for - /// sequencer and replica tasks. - #[miden_instrument( - target = COMPONENT, - )] - pub async fn load( - data_path: &Path, - storage_options: StorageOptions, - ) -> Result { - Self::load_with_database_options(data_path, storage_options, DatabaseOptions::default()) - .await - } - - /// Loads the state from the data directory using explicit database options. - /// - /// The loaded state owns all store data structures and exposes subscription methods for - /// sequencer and replica tasks. - #[miden_instrument( - target = COMPONENT, - )] - pub async fn load_with_database_options( - data_path: &Path, - storage_options: StorageOptions, - database_options: DatabaseOptions, - ) -> Result { - let data_directory = DataDirectory::load(data_path.to_path_buf()) - .map_err(StateInitializationError::DataDirectoryLoadError)?; - - let block_store = Arc::new( - BlockStore::load(data_directory.block_store_dir()) - .map_err(StateInitializationError::BlockStoreLoadError)?, - ); - - let database_filepath = data_directory.database_path(); - let mut db = Db::load_with_pool_size( - database_filepath.clone(), - database_options.connection_pool_size, - ) - .await - .map_err(StateInitializationError::DatabaseLoadError)?; - - let blockchain = load_mmr(&mut db).await?; - let latest_block_num = blockchain.chain_tip().unwrap_or(BlockNumber::GENESIS); - - #[cfg(feature = "rocksdb")] - let (account_storage_config, nullifier_storage_config, forest_storage_config) = ( - storage_options.account_tree.into(), - storage_options.nullifier_tree.into(), - storage_options.account_state_forest.into(), - ); - #[cfg(not(feature = "rocksdb"))] - let (account_storage_config, nullifier_storage_config, forest_storage_config) = { - let _ = &storage_options; - ((), (), ()) - }; - let account_storage = - TreeStorage::create(data_path, &account_storage_config, ACCOUNT_TREE_STORAGE_DIR)?; - let account_tree = account_storage.load_account_tree(&mut db).await?; - - let nullifier_storage = - TreeStorage::create(data_path, &nullifier_storage_config, NULLIFIER_TREE_STORAGE_DIR)?; - let nullifier_tree = nullifier_storage.load_nullifier_tree(&mut db).await?; - - // Verify that tree roots match the expected roots from the database. This catches any - // divergence between persistent storage and the database caused by corruption or incomplete - // shutdown. - verify_tree_consistency(account_tree.root(), nullifier_tree.root(), &mut db).await?; - - let account_tree = AccountTreeWithHistory::new(account_tree, latest_block_num); - - let forest_backend = AccountStateForestBackend::create( - data_path, - &forest_storage_config, - ACCOUNT_STATE_FOREST_STORAGE_DIR, - )?; - let forest = forest_backend.load_account_state_forest(&mut db, latest_block_num).await?; - verify_account_state_forest_consistency(&forest, &mut db).await?; - - let inner = RwLock::new(InnerState { nullifier_tree, blockchain, account_tree }); - - let forest = RwLock::new(forest); - let writer = Mutex::new(()); - let db = Arc::new(db); - - // Initialize the proven tip from the block store. - let proven_tip_init = block_store - .load_proven_tip() - .map_err(StateInitializationError::ProvenTipLoadError)?; - let (proven_tip, _rx) = ProvenTipWriter::new(proven_tip_init); - - // Committed-tip watch: fires after each successful apply_block. - let (committed_tip_tx, _rx) = watch::channel(latest_block_num); - - Ok(Self { - data_directory: data_path.to_path_buf(), - db, - block_store, - inner, - forest, - writer, - proven_tip, - committed_tip_tx, - block_cache: BlockCache::new(BLOCK_CACHE_CAPACITY), - proof_cache: ProofCache::new(PROOF_CACHE_CAPACITY), - }) - } - - /// Returns a watch receiver that wakes every time a new block is committed. - pub fn subscribe_committed_tip(&self) -> watch::Receiver { - self.committed_tip_tx.subscribe() - } - - /// Loads serialized block proving inputs from the block store. - pub async fn load_proving_inputs( - &self, - block_num: BlockNumber, - ) -> std::io::Result>> { - self.block_store.load_proving_inputs(block_num).await - } - - /// Returns a watch receiver that wakes every time the proven-in-sequence tip advances. - pub fn subscribe_proven_tip(&self) -> watch::Receiver { - self.proven_tip.subscribe() - } - - // HELPER FUNCTIONS TO AVOID BLOCKING CALLS IN ASYNC CONTEXT - // -------------------------------------------------------------------------------------------- - - /// Runs a synchronous read-only operation over the inner state on Tokio's blocking path. - /// - /// The account and nullifier trees may be backed by `RocksDB`, so tree access must not run on - /// an async worker thread directly. This helper preserves the current tracing span while - /// moving the blocking lock acquisition and closure body into `block_in_place`. - fn with_inner_read_blocking(&self, f: impl FnOnce(&InnerState) -> R) -> R { - let span = Span::current(); - tokio::task::block_in_place(|| { - span.in_scope(|| { - let inner = self.inner.blocking_read(); - f(&inner) - }) - }) - } - - /// Runs a synchronous mutable operation while holding both in-memory state write locks. - /// - /// Locks are always acquired in inner-state then forest order. Holding both across the database - /// commit window prevents readers from observing canonical tree state from one block and - /// account-state forest data from another. - fn with_inner_and_forest_write_blocking( - &self, - f: impl FnOnce( - &mut InnerState, - &mut AccountStateForest, - ) -> R, - ) -> R { - let span = Span::current(); - tokio::task::block_in_place(|| { - span.in_scope(|| { - let mut inner = self.inner.blocking_write(); - let mut forest = self.forest.blocking_write(); - f(&mut inner, &mut forest) - }) - }) - } - - /// Runs a synchronous read-only operation over the account state forest on Tokio's blocking - /// path. - /// - /// The forest may be backed by `RocksDB`, so accesses to the underlying `LargeSmtForest` must - /// not run directly on an async worker thread. - fn with_forest_read_blocking( - &self, - f: impl FnOnce(&AccountStateForest) -> R, - ) -> R { - let span = Span::current(); - tokio::task::block_in_place(|| { - span.in_scope(|| { - let forest = self.forest.blocking_read(); - f(&forest) - }) - }) - } - - // STATE ACCESSORS - // -------------------------------------------------------------------------------------------- - - /// Queries a [BlockHeader] from the database, and returns it alongside its inclusion proof. - /// - /// If [None] is given as the value of `block_num`, the data for the latest [BlockHeader] is - /// returned. - #[miden_instrument( - level = "debug", - target = COMPONENT, - err, - )] - pub async fn get_block_header( - &self, - block_num: Option, - include_mmr_proof: bool, - ) -> Result<(Option, Option), GetBlockHeaderError> { - let block_header = self.db.select_block_header_by_block_num(block_num).await?; - if let Some(header) = block_header { - let mmr_proof = if include_mmr_proof { - let inner = self.inner.read().await; - let mmr_proof = inner.blockchain.open(header.block_num())?; - Some(mmr_proof) - } else { - None - }; - Ok((Some(header), mmr_proof)) - } else { - Ok((None, None)) - } - } - - /// Queries a list of notes from the database. - /// - /// If the provided list of [`NoteId`] given is empty or no note matches the provided - /// [`NoteId`] an empty list is returned. - pub async fn get_notes_by_id( - &self, - note_ids: Vec, - ) -> Result, DatabaseError> { - self.db.select_notes_by_id(note_ids).await - } - - /// Fetches the inputs for a transaction batch from the database. - /// - /// ## Inputs - /// - /// The function takes as input: - /// - The tx reference blocks are the set of blocks referenced by transactions in the batch. - /// - The unauthenticated note commitments are the set of commitments of unauthenticated notes - /// consumed by all transactions in the batch. For these notes, we attempt to find inclusion - /// proofs. Not all notes will exist in the DB necessarily, as some notes can be created and - /// consumed within the same batch. - /// - /// ## Outputs - /// - /// The function will return: - /// - A block inclusion proof for all tx reference blocks and for all blocks which are - /// referenced by a note inclusion proof. - /// - Note inclusion proofs for all notes that were found in the DB. - /// - The block header that the batch should reference, i.e. the latest known block. - pub async fn get_batch_inputs( - &self, - tx_reference_blocks: BTreeSet, - unauthenticated_note_commitments: BTreeSet, - ) -> Result { - if tx_reference_blocks.is_empty() { - return Err(GetBatchInputsError::TransactionBlockReferencesEmpty); - } - - // First we grab note inclusion proofs for the known notes. These proofs only prove that the - // note was included in a given block. We then also need to prove that each of those blocks - // is included in the chain. - let note_proofs = self - .db - .select_note_inclusion_proofs(unauthenticated_note_commitments) - .await - .map_err(GetBatchInputsError::SelectNoteInclusionProofError)?; - - // The set of blocks that the notes are included in. - let note_blocks = note_proofs.values().map(|proof| proof.location().block_num()); - - // Collect all blocks we need to query without duplicates, which is: - // - all blocks for which we need to prove note inclusion. - // - all blocks referenced by transactions in the batch. - let mut blocks: BTreeSet = tx_reference_blocks; - blocks.extend(note_blocks); - - // Scoped block to automatically drop the read lock guard as soon as we're done. We also - // avoid accessing the db in the block as this would delay dropping the guard. - let (batch_reference_block, partial_mmr) = { - let inner_state = self.inner.read().await; - - let latest_block_num = inner_state.latest_block_num(); - - let highest_block_num = - *blocks.last().expect("we should have checked for empty block references"); - if highest_block_num > latest_block_num { - return Err(GetBatchInputsError::UnknownTransactionBlockReference { - highest_block_num, - latest_block_num, - }); - } - - // Remove the latest block from the to-be-tracked blocks as it will be the reference - // block for the batch itself and thus added to the MMR within the batch kernel, so - // there is no need to prove its inclusion. - blocks.remove(&latest_block_num); - - // SAFETY: - // - The latest block num was retrieved from the inner blockchain from which we will - // also retrieve the proofs, so it is guaranteed to exist in that chain. - // - We have checked that no block number in the blocks set is greater than latest block - // number *and* latest block num was removed from the set. Therefore only block - // numbers smaller than latest block num remain in the set. Therefore all the block - // numbers are guaranteed to exist in the chain state at latest block num. - let partial_mmr = inner_state - .blockchain - .partial_mmr_from_blocks(&blocks, latest_block_num) - .expect("latest block num should exist and all blocks in set should be < than latest block"); - - (latest_block_num, partial_mmr) - }; - - // Fetch the reference block of the batch as part of this query, so we can avoid looking it - // up in a separate DB access. - let mut headers = self - .db - .select_block_headers(blocks.into_iter().chain(std::iter::once(batch_reference_block))) - .await - .map_err(GetBatchInputsError::SelectBlockHeaderError)?; - - // Find and remove the batch reference block as we don't want to add it to the chain MMR. - let header_index = headers - .iter() - .enumerate() - .find_map(|(index, header)| { - (header.block_num() == batch_reference_block).then_some(index) - }) - .expect("DB should have returned the header of the batch reference block"); - - // The order doesn't matter for PartialBlockchain::new, so swap remove is fine. - let batch_reference_block_header = headers.swap_remove(header_index); - - // SAFETY: This should not error because: - // - we're passing exactly the block headers that we've added to the partial MMR, - // - so none of the block headers block numbers should exceed the chain length of the - // partial MMR, - // - and we've added blocks to a BTreeSet, so there can be no duplicates. - // - // We construct headers and partial MMR in concert, so they are consistent. This is why we - // can call the unchecked constructor. - let partial_block_chain = PartialBlockchain::new_unchecked(partial_mmr, headers) - .expect("partial mmr and block headers should be consistent"); - - Ok(BatchInputs { - batch_reference_block_header, - note_proofs, - partial_block_chain, - }) - } - - /// Returns data needed by the block producer to construct and prove the next block. - pub async fn get_block_inputs( - &self, - account_ids: Vec, - nullifiers: Vec, - unauthenticated_note_commitments: BTreeSet, - reference_blocks: BTreeSet, - ) -> Result { - // Get the note inclusion proofs from the DB. We do this first so we have to acquire the - // lock to the state just once. There we need the reference blocks of the note proofs to get - // their authentication paths in the chain MMR. - let unauthenticated_note_proofs = self - .db - .select_note_inclusion_proofs(unauthenticated_note_commitments) - .await - .map_err(GetBlockInputsError::SelectNoteInclusionProofError)?; - - // The set of blocks that the notes are included in. - let note_proof_reference_blocks = - unauthenticated_note_proofs.values().map(|proof| proof.location().block_num()); - - // Collect all blocks we need to prove inclusion for, without duplicates. - let mut blocks = reference_blocks; - blocks.extend(note_proof_reference_blocks); - - let (latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr) = - self.get_block_inputs_witnesses(&mut blocks, &account_ids, &nullifiers)?; - - // Fetch the block headers for all blocks in the partial MMR plus the latest one which will - // be used as the previous block header of the block being built. - let mut headers = self - .db - .select_block_headers(blocks.into_iter().chain(std::iter::once(latest_block_number))) - .await - .map_err(GetBlockInputsError::SelectBlockHeaderError)?; - - // Find and remove the latest block as we must not add it to the chain MMR, since it is not - // yet in the chain. - let latest_block_header_index = headers - .iter() - .enumerate() - .find_map(|(index, header)| { - (header.block_num() == latest_block_number).then_some(index) - }) - .expect("DB should have returned the header of the latest block header"); - - // The order doesn't matter for PartialBlockchain::new, so swap remove is fine. - let latest_block_header = headers.swap_remove(latest_block_header_index); - - // SAFETY: This should not error because: - // - we're passing exactly the block headers that we've added to the partial MMR, - // - so none of the block header's block numbers should exceed the chain length of the - // partial MMR, - // - and we've added blocks to a BTreeSet, so there can be no duplicates. - // - // We construct headers and partial MMR in concert, so they are consistent. This is why we - // can call the unchecked constructor. - let partial_block_chain = PartialBlockchain::new_unchecked(partial_mmr, headers) - .expect("partial mmr and block headers should be consistent"); - - Ok(BlockInputs::new( - latest_block_header, - partial_block_chain, - account_witnesses, - nullifier_witnesses, - unauthenticated_note_proofs, - )) - } - - /// Get account and nullifier witnesses for the requested account IDs and nullifier as well as - /// the [`PartialMmr`] for the given blocks. The MMR won't contain the latest block and its - /// number is removed from `blocks` and returned separately. - /// - /// This method acquires the lock to the inner state and does not access the DB so we release - /// the lock asap. - fn get_block_inputs_witnesses( - &self, - blocks: &mut BTreeSet, - account_ids: &[AccountId], - nullifiers: &[Nullifier], - ) -> Result { - self.with_inner_read_blocking(|inner| { - let latest_block_number = inner.latest_block_num(); - - // If `blocks` is empty, use the latest block number which will never trigger the error. - let highest_block_number = blocks.last().copied().unwrap_or(latest_block_number); - if highest_block_number > latest_block_number { - return Err(GetBlockInputsError::UnknownBatchBlockReference { - highest_block_number, - latest_block_number, - }); - } - - // The latest block is not yet in the chain MMR, so we can't (and don't need to) prove - // its inclusion in the chain. - blocks.remove(&latest_block_number); - - // Fetch the partial MMR at the state of the latest block with authentication paths for - // the provided set of blocks. - // - // SAFETY: - // - The latest block num was retrieved from the inner blockchain from which we will - // also retrieve the proofs, so it is guaranteed to exist in that chain. - // - We have checked that no block number in the blocks set is greater than latest block - // number *and* latest block num was removed from the set. Therefore only block - // numbers smaller than latest block num remain in the set. Therefore all the block - // numbers are guaranteed to exist in the chain state at latest block num. - let partial_mmr = - inner.blockchain.partial_mmr_from_blocks(blocks, latest_block_number).expect( - "latest block num should exist and all blocks in set should be < than latest block", - ); - - // Fetch witnesses for all accounts. - let account_witnesses = account_ids - .iter() - .copied() - .map(|account_id| (account_id, inner.account_tree.open_latest(account_id))) - .collect::>(); - - // Fetch witnesses for all nullifiers. We don't check whether the nullifiers are spent - // or not as this is done as part of proposing the block. - let nullifier_witnesses: BTreeMap = nullifiers - .iter() - .copied() - .map(|nullifier| (nullifier, inner.nullifier_tree.open(&nullifier))) - .collect(); - - Ok((latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr)) - }) - } - - /// Returns data needed by the block producer to verify transactions validity. - #[miden_instrument( - target = COMPONENT, - fields( - account.id=%account_id, - nullifiers = %format_array(nullifiers), - ), - )] - pub async fn get_transaction_inputs( - &self, - account_id: AccountId, - nullifiers: &[Nullifier], - unauthenticated_note_commitments: Vec, - ) -> Result { - let tree_inputs = self.with_inner_read_blocking(|inner| { - let account_commitment = inner.account_tree.get_latest_commitment(account_id); - - let new_account_id_prefix_is_unique = if account_commitment.is_empty() { - Some(!inner.account_tree.contains_account_id_prefix_in_latest(account_id.prefix())) - } else { - None - }; - - // Non-unique account Id prefixes for new accounts are not allowed. - if let Some(false) = new_account_id_prefix_is_unique { - return Err(TransactionInputs { - new_account_id_prefix_is_unique, - ..Default::default() - }); - } - - let nullifiers = nullifiers - .iter() - .map(|nullifier| NullifierInfo { - nullifier: *nullifier, - block_num: inner.nullifier_tree.get_block_num(nullifier).unwrap_or_default(), - }) - .collect(); - - Ok((account_commitment, nullifiers, new_account_id_prefix_is_unique)) - }); - let (account_commitment, nullifiers, new_account_id_prefix_is_unique) = match tree_inputs { - Ok(inputs) => inputs, - Err(inputs) => return Ok(inputs), - }; - - let found_unauthenticated_notes = self - .db - .select_existing_note_commitments(unauthenticated_note_commitments) - .await?; - - Ok(TransactionInputs { - account_commitment, - nullifiers, - found_unauthenticated_notes, - new_account_id_prefix_is_unique, - }) - } - - /// Filters `account_ids` down to the subset classified as network accounts. - pub async fn filter_network_accounts( - &self, - account_ids: &[AccountId], - ) -> Result, DatabaseError> { - self.db.select_network_accounts_subset(account_ids.to_vec()).await - } - - /// Returns the effective chain tip for the given finality level. - /// - /// - [`Finality::Committed`]: returns the latest committed block number (from in-memory MMR). - /// - [`Finality::Proven`]: returns the latest proven-in-sequence block number (cached via watch - /// channel, updated by the proof scheduler). - pub async fn chain_tip(&self, finality: Finality) -> BlockNumber { - match finality { - Finality::Committed => self - .inner - .read() - .instrument(tracing::info_span!("acquire_inner")) - .await - .latest_block_num(), - Finality::Proven => self.proven_tip.read(), - } - } - - /// Loads a block from the in-memory replica cache or block store. Return `Ok(None)` if the - /// block is not found. - pub async fn load_block( - &self, - block_num: BlockNumber, - ) -> Result>, DatabaseError> { - if block_num > self.chain_tip(Finality::Committed).await { - return Ok(None); - } - if let Some(block) = self.block_cache.get(block_num) { - return Ok(Some(block.block_bytes().to_vec())); - } - self.block_store.load_block(block_num).await.map_err(Into::into) - } - - /// Loads a block proof from the in-memory replica cache or block store. Returns `Ok(None)` if - /// the proof is not found. - pub async fn load_proof( - &self, - block_num: BlockNumber, - ) -> Result>, DatabaseError> { - if block_num > self.chain_tip(Finality::Proven).await { - return Ok(None); - } - if let Some(proof) = self.proof_cache.get(block_num) { - return Ok(Some(proof.proof_bytes().to_vec())); - } - self.block_store.load_proof(block_num).await.map_err(Into::into) - } - - /// Returns the script for a note by its root. - pub async fn get_note_script_by_root( - &self, - root: Word, - ) -> Result, DatabaseError> { - self.db.select_note_script_by_root(root).await - } -} diff --git a/crates/store/src/state/replica.rs b/crates/store/src/state/replica.rs index 61533de8b1..e24d3acffb 100644 --- a/crates/store/src/state/replica.rs +++ b/crates/store/src/state/replica.rs @@ -1,8 +1,24 @@ +//! Block and proof serving for replica subscriptions. +//! +//! Replica subscribers stream committed blocks and block proofs from this node. The writer pushes +//! each freshly committed block (and the proof path each proven block) into a FIFO cache here, so +//! subscribers keeping up with the tip are served from memory; a subscriber that has fallen +//! behind the cache window falls back to reading the block store. +//! +//! These reads serve raw block/proof bytes from the caches and the block store — they never touch +//! the database or tree state, which is why they live on [`State`] directly rather than on a +//! [`StateView`](super::StateView). The tip checks gating them are live availability checks +//! ("is this block committed/proven yet?"), deliberately race-tolerant: a `None` for a block that +//! commits an instant later is corrected by the next tip-watch wakeup. + use std::sync::Arc; use miden_node_utils::block_cache::BlockOrderedCache; use miden_protocol::block::BlockNumber; +use crate::errors::DatabaseError; +use crate::state::State; + // BLOCK NOTIFICATION // ================================================================================================ @@ -65,3 +81,46 @@ pub type BlockCache = BlockOrderedCache; /// FIFO cache of recent block proofs for replica subscriptions. pub type ProofCache = BlockOrderedCache; + +// REPLICA STATE ACCESS +// ================================================================================================ + +impl State { + /// Loads a block from the in-memory replica cache or block store. Return `Ok(None)` if the + /// block is not found. + pub async fn load_block( + &self, + block_num: BlockNumber, + ) -> Result>, DatabaseError> { + if block_num > self.committed_tip() { + return Ok(None); + } + if let Some(block) = self.block_cache.get(block_num) { + return Ok(Some(block.block_bytes().to_vec())); + } + self.block_store.load_block(block_num).await.map_err(Into::into) + } + + /// Loads a block proof from the in-memory replica cache or block store. Returns `Ok(None)` if + /// the proof is not found. + pub async fn load_proof( + &self, + block_num: BlockNumber, + ) -> Result>, DatabaseError> { + if block_num > self.proven_tip() { + return Ok(None); + } + if let Some(proof) = self.proof_cache.get(block_num) { + return Ok(Some(proof.proof_bytes().to_vec())); + } + self.block_store.load_proof(block_num).await.map_err(Into::into) + } + + /// Loads serialized block proving inputs from the block store. + pub async fn load_proving_inputs( + &self, + block_num: BlockNumber, + ) -> std::io::Result>> { + self.block_store.load_proving_inputs(block_num).await + } +} diff --git a/crates/store/src/state/tip.rs b/crates/store/src/state/tip.rs new file mode 100644 index 0000000000..05d823905b --- /dev/null +++ b/crates/store/src/state/tip.rs @@ -0,0 +1,63 @@ +//! Live chain-tip queries and tip subscriptions, deliberately kept off +//! [`StateView`](super::StateView). +//! +//! Both tips are published through watch channels by their single writers (the block writer for +//! the committed tip, the proof scheduler or proof sync for the proven tip). Everything here +//! reads or subscribes to those channels; none of it touches state snapshots — trees, forest, or +//! DB — so these values advance independently of any `StateView`. +//! +//! This lives on [`State`] rather than `StateView` on purpose, not just because the values are +//! live. A `StateView` pins a whole `StateSnapshot` (including the `RocksDB` snapshots backing +//! the trees) and is meant to be released as soon as possible; a tip read never needs that +//! snapshot, so routing it through a view would pin one for no reason. The `subscribe_*` +//! receivers go further: they are meant to be held for the lifetime of a long-running task (a +//! proof-sync loop, a subscription stream), which is the opposite of a `StateView`'s +//! request-scoped, drop-it-immediately lifetime — so they could not live on `StateView` even if +//! the values themselves needed one. + +use miden_protocol::block::BlockNumber; +use tokio::sync::watch; + +use super::State; + +// TIP QUERIES & SUBSCRIPTIONS +// ================================================================================================ + +impl State { + /// Returns the latest committed (but not necessarily proven) block number. + /// + /// This is a live value: it advances independently of any [`StateView`](super::StateView). + /// Reads that must be consistent with data must use a view's tip instead. + /// + /// Ordering matters when combining this with a view: read the tip *before* creating the view. + /// Tips are monotonic and the committed tip is published after its snapshot, so a view + /// created afterwards can always serve a tip read earlier. Reading a live tip *after* + /// creating a view is racy — the tip may have advanced past the view's pinned snapshot. + pub fn committed_tip(&self) -> BlockNumber { + *self.committed_tip_tx.borrow() + } + + /// Returns the latest block number proven in an unbroken sequence from genesis. + /// + /// This is a live value published by the proof scheduler (sequencer mode) or the proof sync + /// loop (full-node mode); it is always at or behind the committed tip. + /// + /// Ordering matters when combining this with a view: read the tip *before* creating the view. + /// Proofs are only applied to committed blocks and each committed tip is published after its + /// snapshot, so a view created afterwards can always serve a proven tip read earlier. Reading + /// it *after* creating a view is racy — a block may commit and prove concurrently, putting + /// the proven tip past the view's pinned snapshot. + pub fn proven_tip(&self) -> BlockNumber { + self.proven_tip.read() + } + + /// Returns a watch receiver that wakes every time a new block is committed. + pub fn subscribe_committed_tip(&self) -> watch::Receiver { + self.committed_tip_tx.subscribe() + } + + /// Returns a watch receiver that wakes every time the proven-in-sequence tip advances. + pub fn subscribe_proven_tip(&self) -> watch::Receiver { + self.proven_tip.subscribe() + } +} diff --git a/crates/store/src/state/view/account/mod.rs b/crates/store/src/state/view/account/mod.rs new file mode 100644 index 0000000000..e84ccc56e0 --- /dev/null +++ b/crates/store/src/state/view/account/mod.rs @@ -0,0 +1,412 @@ +use std::collections::HashSet; + +use miden_node_proto::domain::account::{ + AccountDetailRequest, + AccountDetails, + AccountRequest, + AccountResponse, + AccountStorageDetails, + AccountStorageMapDetails, + AccountStorageRequest, + AccountVaultDetails, + SlotData, + StorageMapEntries, + StorageMapRequest, +}; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::account::{AccountId, AccountStorageHeader, StorageSlotName, StorageSlotType}; +use miden_protocol::block::BlockNumber; +use miden_protocol::block::account_tree::AccountWitness; + +use super::{ScopedBlockNum, StateView}; +use crate::COMPONENT; +use crate::account_state_forest::AccountStorageMapResult; +use crate::errors::{DatabaseError, GetAccountError}; + +impl StateView { + /// Returns an account witness and optionally account details at a specific block. + /// + /// The witness is a Merkle proof of inclusion in the account tree, proving the account's + /// state commitment. If `details` is requested, the method also returns the account's code, + /// vault assets, and storage data. Account details are only available for public accounts. + /// + /// If `block_num` is provided, returns the state at that historical block; otherwise, returns + /// the latest state. Note that historical states are only available for recent blocks close + /// to the chain tip. + #[miden_instrument( + target = COMPONENT, + )] + pub async fn get_account( + &self, + account_request: AccountRequest, + ) -> Result { + let AccountRequest { block_num, account_id, details } = account_request; + + if details.is_some() && !account_id.is_public() { + return Err(GetAccountError::AccountNotPublic(account_id)); + } + + let (scoped_block, witness) = self.get_account_witness(block_num, account_id).await?; + + let details = if let Some(request) = details { + Some( + self.fetch_public_account_details(account_id, scoped_block, &witness, request) + .await?, + ) + } else { + None + }; + + Ok(AccountResponse { + block_num: *scoped_block, + witness, + details, + }) + } + + /// Returns an account witness (Merkle proof of inclusion in the account tree) together with + /// the resolved block as a scoped block number. + /// + /// If `block_num` is provided, returns the witness at that historical block; otherwise, + /// returns the witness at the latest block. The tree resolution doubles as the tip + /// validation, so the returned block is ready for block-bounded database queries. + #[miden_instrument( + target = COMPONENT, + )] + async fn get_account_witness( + &self, + block_num: Option, + account_id: AccountId, + ) -> Result<(ScopedBlockNum, AccountWitness), GetAccountError> { + // Historical query: scope the requested block up front — a block beyond this view's tip is + // unknown, so a missing tree entry below can only mean it was pruned. + if let Some(requested_block) = block_num { + let scoped_block = self + .scope_block(requested_block) + .ok_or(GetAccountError::UnknownBlock(requested_block))?; + let witness = self + .with_inner_read_blocking(|inner_state| { + inner_state.account_tree.open_at(account_id, *scoped_block) + }) + .ok_or(GetAccountError::BlockPruned(*scoped_block))?; + Ok((scoped_block, witness)) + } else { + // Latest query: the tree's latest state is the view's tip. + let witness = self.with_inner_read_blocking(|inner_state| { + inner_state.account_tree.open_latest(account_id) + }); + Ok((self.tip(), witness)) + } + } + + /// Returns storage map details from the forest for a specific account and storage slot. + /// + /// The forest can only be used if all hashed keys in the storage map are known in the + /// reverse-key LRU cache. If any hashed key is unknown, the method returns `Ok(None)` to signal + /// that the caller should fall back to reconstructing the storage map details from the + /// database. + #[miden_instrument( + target = COMPONENT, + )] + fn get_storage_map_details_from_forest( + &self, + account_id: AccountId, + slot_name: &StorageSlotName, + block_num: ScopedBlockNum, + ) -> Result, DatabaseError> { + self.with_forest_read_blocking(|forest| { + match forest + .get_storage_map_details_for_all_entries(account_id, slot_name.clone(), *block_num) + .map_err(DatabaseError::MerkleError)? + { + AccountStorageMapResult::NotFound => Err(DatabaseError::StorageRootNotFound { + account_id, + slot_name: slot_name.to_string(), + block_num: *block_num, + }), + AccountStorageMapResult::Details(details) => Ok(Some(details)), + AccountStorageMapResult::CannotReconstructKeysFromCache => Ok(None), + } + }) + } + + /// Returns vault details by reconstructing the vault from the database. + async fn reconstruct_vault_details_from_db( + &self, + account_id: AccountId, + block_num: ScopedBlockNum, + ) -> Result { + let assets = self.db.select_account_vault_at_block(account_id, block_num).await?; + + if assets.len() > AccountVaultDetails::MAX_RETURN_ENTRIES { + return Ok(AccountVaultDetails::LimitExceeded); + } + + let keys = assets.iter().map(miden_protocol::asset::Asset::id); + + // The reverse-key caches are shared between the writer and all snapshots, so caching via + // the current snapshot's forest is visible everywhere. + self.with_forest_read_blocking(|forest| { + forest + .vault_key_cache + .put_many(keys.into_iter().map(|raw_key| (raw_key.hash(), raw_key))); + }); + + Ok(AccountVaultDetails::from_assets(assets)) + } + + /// Returns storage map details by reconstructing the storage map from the database. + async fn reconstruct_storage_map_details_from_db( + &self, + account_id: AccountId, + slot_name: StorageSlotName, + block_num: ScopedBlockNum, + ) -> Result { + let details = self + .db + .reconstruct_storage_map_from_db( + account_id, + slot_name, + block_num, + Some(AccountStorageMapDetails::MAX_RETURN_ENTRIES), + ) + .await?; + + if let StorageMapEntries::AllEntries(entries) = &details.entries { + self.with_forest_read_blocking(|forest| { + forest.cache_storage_map_keys(entries.iter().map(|(raw_key, _)| *raw_key)); + }); + } + + Ok(details) + } + + /// Fetches the account details (code, vault, storage) for a public account at the specified + /// block. + /// + /// This method queries the database to fetch the account state and processes the detail + /// request to return only the requested information. + /// + /// For specific key queries (`SlotData::MapKeys`), the forest is used to provide SMT proofs. + /// Returns an error if the forest doesn't have data for the requested slot. + /// All-entries queries (`SlotData::All`) use the forest when all hashed keys are known in the + /// reverse-key LRU cache, otherwise they fall back to database reconstruction. + #[miden_instrument( + target = COMPONENT, + )] + async fn fetch_public_account_details( + &self, + account_id: AccountId, + scoped_block: ScopedBlockNum, + witness: &AccountWitness, + detail_request: AccountDetailRequest, + ) -> Result { + let AccountDetailRequest { + code_commitment, + asset_vault_commitment, + storage_request, + } = detail_request; + + if !account_id.is_public() { + return Err(GetAccountError::AccountNotPublic(account_id)); + } + + // Query account header and storage header together in a single DB call + let (account_header, storage_header) = self + .db + .select_account_header_with_storage_header_at_block(account_id, scoped_block) + .await? + .ok_or(GetAccountError::AccountNotFound(account_id, *scoped_block))?; + + let should_apply_response_budget = + matches!(&storage_request, AccountStorageRequest::AllStorageMaps); + let storage_requests = expand_account_storage_request(storage_request, &storage_header); + + let account_code = match code_commitment { + Some(commitment) if commitment == account_header.code_commitment() => None, + Some(_) => { + self.db + .select_account_code_by_commitment(account_header.code_commitment()) + .await? + }, + None => None, + }; + + // Query account state forest for vault details on commitment mismatch. + // + // The forest can only reconstruct the vault if all hashed vault keys are known in the + // reverse-key LRU cache. If any hashed key is unknown, the forest returns `None` and we + // fall back to reconstructing the vault details from the database. + let vault_details = match asset_vault_commitment { + Some(commitment) if commitment == account_header.vault_root() => { + AccountVaultDetails::empty() + }, + Some(_) => { + let forest_details = self.with_forest_read_blocking(|forest| { + forest.get_vault_details(account_id, *scoped_block).map_err(|err| { + DatabaseError::DataCorrupted(format!( + "failed to reconstruct vault for account {account_id} at block {}: {err}", + *scoped_block, + )) + }) + })?; + + match forest_details { + Some(details) => details, + None => { + self.reconstruct_vault_details_from_db(account_id, scoped_block).await? + }, + } + }, + None => AccountVaultDetails::empty(), + }; + + // Split storage map requests into two categories: + // - slots with explicit keys (including proofs) + // - slots with "all entries" + let mut storage_map_details = + Vec::::with_capacity(storage_requests.len()); + let mut map_keys_requests = Vec::new(); + let mut all_entries_requests = Vec::new(); + let mut storage_request_slots = Vec::with_capacity(storage_requests.len()); + + for (index, StorageMapRequest { slot_name, slot_data }) in + storage_requests.into_iter().enumerate() + { + storage_request_slots.push(slot_name.clone()); + match slot_data { + SlotData::MapKeys(keys) => { + map_keys_requests.push((index, slot_name, keys)); + }, + SlotData::All => { + all_entries_requests.push((index, slot_name)); + }, + } + } + + let mut storage_map_details_by_index = vec![None; storage_request_slots.len()]; + + // Handle slots with explicit key requests + if !map_keys_requests.is_empty() { + self.with_forest_read_blocking(|forest| { + for (index, slot_name, keys) in map_keys_requests { + let details = forest + .get_storage_map_details_for_keys( + account_id, + slot_name.clone(), + *scoped_block, + &keys, + ) + .ok_or_else(|| DatabaseError::StorageRootNotFound { + account_id, + slot_name: slot_name.to_string(), + block_num: *scoped_block, + })? + .map_err(DatabaseError::MerkleError)?; + storage_map_details_by_index[index] = Some(details); + } + Ok::<(), DatabaseError>(()) + })?; + } + + // Handle slots with "all entries" requests + for (index, slot_name) in all_entries_requests { + let details = match self.get_storage_map_details_from_forest( + account_id, + &slot_name, + scoped_block, + )? { + Some(details) => details, + None => { + self.reconstruct_storage_map_details_from_db( + account_id, + slot_name, + scoped_block, + ) + .await? + }, + }; + storage_map_details_by_index[index] = Some(details); + } + + for (details, slot_name) in + storage_map_details_by_index.into_iter().zip(storage_request_slots.iter()) + { + let details = details.ok_or_else(|| DatabaseError::StorageRootNotFound { + account_id, + slot_name: slot_name.to_string(), + block_num: *scoped_block, + })?; + storage_map_details.push(details); + } + + // In case of an "all storage maps" request we have to be careful: even with the per-slot + // limit of [`AccountStorageMapDetails::MAX_RETURN_ENTRIES`] we might go over the response + // size limit. Here we make sure that we're within that limit by potentially truncating the + // response. + if should_apply_response_budget { + return Ok(apply_all_storage_maps_response_budget( + *scoped_block, + witness, + account_header, + account_code, + vault_details, + storage_header, + storage_map_details, + storage_request_slots, + MAX_ALL_STORAGE_MAPS_RESPONSE_PAYLOAD_WITH_BUDGET_RESERVED_FOR_LIMIT_EXCEEDED_SLOTS, + )); + } + + Ok(AccountDetails { + account_header, + account_code, + vault_details, + storage_details: AccountStorageDetails { + header: storage_header, + map_details: storage_map_details, + }, + }) + } +} + +// HELPERS +// ================================================================================================ + +/// Expand [`AccountStorageRequest`] to a vector of slot requests. +fn expand_account_storage_request( + storage_request: AccountStorageRequest, + storage_header: &AccountStorageHeader, +) -> Vec { + match storage_request { + AccountStorageRequest::None => Vec::new(), + AccountStorageRequest::Explicit(requests) => requests, + AccountStorageRequest::AllStorageMaps => storage_header + .slots() + .filter(|slot| slot.slot_type() == StorageSlotType::Map) + .map(|slot| StorageMapRequest { + slot_name: slot.name().clone(), + slot_data: SlotData::All, + }) + .collect(), + } +} + +mod response_budget; +use response_budget::{ + MAX_ALL_STORAGE_MAPS_RESPONSE_PAYLOAD_WITH_BUDGET_RESERVED_FOR_LIMIT_EXCEEDED_SLOTS, + apply_all_storage_maps_response_budget, +}; + +// NETWORK ACCOUNT CLASSIFICATION +// ================================================================================================ + +impl StateView { + /// Filters `account_ids` down to the subset classified as network accounts. + pub async fn filter_network_accounts( + &self, + account_ids: &[AccountId], + ) -> Result, DatabaseError> { + self.db.select_network_accounts_subset(account_ids.to_vec()).await + } +} diff --git a/crates/store/src/state/account.rs b/crates/store/src/state/view/account/response_budget.rs similarity index 50% rename from crates/store/src/state/account.rs rename to crates/store/src/state/view/account/response_budget.rs index d8dd4bf40d..b8ecaa6998 100644 --- a/crates/store/src/state/account.rs +++ b/crates/store/src/state/view/account/response_budget.rs @@ -1,405 +1,25 @@ +//! Response-size budgeting for `all-storage-maps` account detail responses. +//! +//! Even with the per-slot entry limit, an `all-storage-maps` request can exceed the response +//! payload cap. The budget below keeps the encoded response within the cap by replacing map +//! contents with "limit exceeded" markers once the budget is spent, reserving space for those +//! markers up front. + use miden_node_proto::domain::account::{ - AccountDetailRequest, AccountDetails, - AccountRequest, AccountResponse, AccountStorageDetails, AccountStorageMapDetails, - AccountStorageRequest, AccountVaultDetails, - SlotData, StorageMapEntries, - StorageMapRequest, }; use miden_node_proto::generated as proto; use miden_node_proto::prost::Message as _; use miden_node_proto::prost::encoding::{encoded_len_varint, key_len}; use miden_node_utils::limiter::MAX_RESPONSE_PAYLOAD_BYTES; -use miden_node_utils::tracing::miden_instrument; -use miden_protocol::account::{ - AccountHeader, - AccountId, - AccountStorageHeader, - StorageSlotName, - StorageSlotType, -}; +use miden_protocol::account::{AccountHeader, AccountStorageHeader, StorageSlotName}; use miden_protocol::block::BlockNumber; use miden_protocol::block::account_tree::AccountWitness; -use tracing::Instrument; - -use super::State; -use crate::COMPONENT; -use crate::account_state_forest::AccountStorageMapResult; -use crate::errors::{DatabaseError, GetAccountError}; - -impl State { - /// Returns an account witness and optionally account details at a specific block. - /// - /// The witness is a Merkle proof of inclusion in the account tree, proving the account's - /// state commitment. If `details` is requested, the method also returns the account's code, - /// vault assets, and storage data. Account details are only available for public accounts. - /// - /// If `block_num` is provided, returns the state at that historical block; otherwise, returns - /// the latest state. Note that historical states are only available for recent blocks close - /// to the chain tip. - #[miden_instrument( - target = COMPONENT, - )] - pub async fn get_account( - &self, - account_request: AccountRequest, - ) -> Result { - let AccountRequest { block_num, account_id, details } = account_request; - - if details.is_some() && !account_id.is_public() { - return Err(GetAccountError::AccountNotPublic(account_id)); - } - - let (block_num, witness) = self.get_account_witness(block_num, account_id).await?; - - let details = if let Some(request) = details { - Some( - self.fetch_public_account_details(account_id, block_num, &witness, request) - .await?, - ) - } else { - None - }; - - Ok(AccountResponse { block_num, witness, details }) - } - - /// Returns an account witness (Merkle proof of inclusion in the account tree). - /// - /// If `block_num` is provided, returns the witness at that historical block; - /// otherwise, returns the witness at the latest block. - #[miden_instrument( - target = COMPONENT, - )] - async fn get_account_witness( - &self, - block_num: Option, - account_id: AccountId, - ) -> Result<(BlockNumber, AccountWitness), GetAccountError> { - self.with_inner_read_blocking(|inner_state| { - // Determine which block to query - let (block_num, witness) = if let Some(requested_block) = block_num { - // Historical query: use the account tree with history - let witness = inner_state - .account_tree - .open_at(account_id, requested_block) - .ok_or_else(|| { - let latest_block = inner_state.account_tree.block_number_latest(); - if requested_block > latest_block { - GetAccountError::UnknownBlock(requested_block) - } else { - GetAccountError::BlockPruned(requested_block) - } - })?; - (requested_block, witness) - } else { - // Latest query: use the latest state - let block_num = inner_state.account_tree.block_number_latest(); - let witness = inner_state.account_tree.open_latest(account_id); - (block_num, witness) - }; - - Ok((block_num, witness)) - }) - } - - /// Returns storage map details from the forest for a specific account and storage slot. - /// - /// The forest can only be used if all hashed keys in the storage map are known in the - /// reverse-key LRU cache. If any hashed key is unknown, the method returns `Ok(None)` to signal - /// that the caller should fall back to reconstructing the storage map details from the - /// database. - #[miden_instrument( - target = COMPONENT, - )] - fn get_storage_map_details_from_forest( - &self, - account_id: AccountId, - slot_name: &StorageSlotName, - block_num: BlockNumber, - ) -> Result, DatabaseError> { - self.with_forest_read_blocking(|forest| { - match forest - .get_storage_map_details_for_all_entries(account_id, slot_name.clone(), block_num) - .map_err(DatabaseError::MerkleError)? - { - AccountStorageMapResult::NotFound => Err(DatabaseError::StorageRootNotFound { - account_id, - slot_name: slot_name.to_string(), - block_num, - }), - AccountStorageMapResult::Details(details) => Ok(Some(details)), - AccountStorageMapResult::CannotReconstructKeysFromCache => Ok(None), - } - }) - } - - /// Returns vault details by reconstructing the vault from the database. - async fn reconstruct_vault_details_from_db( - &self, - account_id: AccountId, - block_num: BlockNumber, - ) -> Result { - let assets = self.db.select_account_vault_at_block(account_id, block_num).await?; - - if assets.len() > AccountVaultDetails::MAX_RETURN_ENTRIES { - return Ok(AccountVaultDetails::LimitExceeded); - } - - let keys = assets.iter().map(miden_protocol::asset::Asset::id); - - let forest = self.forest.write().await; - - forest - .vault_key_cache - .put_many(keys.into_iter().map(|raw_key| (raw_key.hash(), raw_key))); - - Ok(AccountVaultDetails::from_assets(assets)) - } - - /// Returns storage map details by reconstructing the storage map from the database. - async fn reconstruct_storage_map_details_from_db( - &self, - account_id: AccountId, - slot_name: StorageSlotName, - block_num: BlockNumber, - ) -> Result { - let details = self - .db - .reconstruct_storage_map_from_db( - account_id, - slot_name, - block_num, - Some(AccountStorageMapDetails::MAX_RETURN_ENTRIES), - ) - .await?; - - if let StorageMapEntries::AllEntries(entries) = &details.entries { - self.forest - .write() - .await - .cache_storage_map_keys(entries.iter().map(|(raw_key, _)| *raw_key)); - } - - Ok(details) - } - - /// Fetches the account details (code, vault, storage) for a public account at the specified - /// block. - /// - /// This method queries the database to fetch the account state and processes the detail - /// request to return only the requested information. - /// - /// For specific key queries (`SlotData::MapKeys`), the forest is used to provide SMT proofs. - /// Returns an error if the forest doesn't have data for the requested slot. - /// All-entries queries (`SlotData::All`) use the forest when all hashed keys are known in the - /// reverse-key LRU cache, otherwise they fall back to database reconstruction. - #[miden_instrument( - target = COMPONENT, - )] - async fn fetch_public_account_details( - &self, - account_id: AccountId, - block_num: BlockNumber, - witness: &AccountWitness, - detail_request: AccountDetailRequest, - ) -> Result { - let AccountDetailRequest { - code_commitment, - asset_vault_commitment, - storage_request, - } = detail_request; - - if !account_id.is_public() { - return Err(GetAccountError::AccountNotPublic(account_id)); - } - - // Validate block exists in the blockchain before querying the database - { - let inner = self.inner.read().instrument(tracing::info_span!("acquire_inner")).await; - let latest_block_num = inner.latest_block_num(); - - if block_num > latest_block_num { - return Err(GetAccountError::UnknownBlock(block_num)); - } - } - - // Query account header and storage header together in a single DB call - let (account_header, storage_header) = self - .db - .select_account_header_with_storage_header_at_block(account_id, block_num) - .await? - .ok_or(GetAccountError::AccountNotFound(account_id, block_num))?; - - let should_apply_response_budget = - matches!(&storage_request, AccountStorageRequest::AllStorageMaps); - let storage_requests = expand_account_storage_request(storage_request, &storage_header); - - let account_code = match code_commitment { - Some(commitment) if commitment == account_header.code_commitment() => None, - Some(_) => { - self.db - .select_account_code_by_commitment(account_header.code_commitment()) - .await? - }, - None => None, - }; - - // Query account state forest for vault details on commitment mismatch. - // - // The forest can only reconstruct the vault if all hashed vault keys are known in the - // reverse-key LRU cache. If any hashed key is unknown, the forest returns `None` and we - // fall back to reconstructing the vault details from the database. - let vault_details = match asset_vault_commitment { - Some(commitment) if commitment == account_header.vault_root() => { - AccountVaultDetails::empty() - }, - Some(_) => { - let forest_details = self.with_forest_read_blocking(|forest| { - forest.get_vault_details(account_id, block_num).map_err(|err| { - DatabaseError::DataCorrupted(format!( - "failed to reconstruct vault for account {account_id} at block {block_num}: {err}" - )) - }) - })?; - - match forest_details { - Some(details) => details, - None => self.reconstruct_vault_details_from_db(account_id, block_num).await?, - } - }, - None => AccountVaultDetails::empty(), - }; - - // Split storage map requests into two categories: - // - slots with explicit keys (including proofs) - // - slots with "all entries" - let mut storage_map_details = - Vec::::with_capacity(storage_requests.len()); - let mut map_keys_requests = Vec::new(); - let mut all_entries_requests = Vec::new(); - let mut storage_request_slots = Vec::with_capacity(storage_requests.len()); - - for (index, StorageMapRequest { slot_name, slot_data }) in - storage_requests.into_iter().enumerate() - { - storage_request_slots.push(slot_name.clone()); - match slot_data { - SlotData::MapKeys(keys) => { - map_keys_requests.push((index, slot_name, keys)); - }, - SlotData::All => { - all_entries_requests.push((index, slot_name)); - }, - } - } - - let mut storage_map_details_by_index = vec![None; storage_request_slots.len()]; - - // Handle slots with explicit key requests - if !map_keys_requests.is_empty() { - self.with_forest_read_blocking(|forest| { - for (index, slot_name, keys) in map_keys_requests { - let details = forest - .get_storage_map_details_for_keys( - account_id, - slot_name.clone(), - block_num, - &keys, - ) - .ok_or_else(|| DatabaseError::StorageRootNotFound { - account_id, - slot_name: slot_name.to_string(), - block_num, - })? - .map_err(DatabaseError::MerkleError)?; - storage_map_details_by_index[index] = Some(details); - } - Ok::<(), DatabaseError>(()) - })?; - } - - // Handle slots with "all entries" requests - for (index, slot_name) in all_entries_requests { - let details = match self - .get_storage_map_details_from_forest(account_id, &slot_name, block_num)? - { - Some(details) => details, - None => { - self.reconstruct_storage_map_details_from_db(account_id, slot_name, block_num) - .await? - }, - }; - storage_map_details_by_index[index] = Some(details); - } - - for (details, slot_name) in - storage_map_details_by_index.into_iter().zip(storage_request_slots.iter()) - { - let details = details.ok_or_else(|| DatabaseError::StorageRootNotFound { - account_id, - slot_name: slot_name.to_string(), - block_num, - })?; - storage_map_details.push(details); - } - - // In case of an "all storage maps" request we have to be careful: even with the per-slot - // limit of [`AccountStorageMapDetails::MAX_RETURN_ENTRIES`] we might go over the response - // size limit. Here we make sure that we're within that limit by potentially truncating the - // response. - if should_apply_response_budget { - return Ok(apply_all_storage_maps_response_budget( - block_num, - witness, - account_header, - account_code, - vault_details, - storage_header, - storage_map_details, - storage_request_slots, - MAX_ALL_STORAGE_MAPS_RESPONSE_PAYLOAD_WITH_BUDGET_RESERVED_FOR_LIMIT_EXCEEDED_SLOTS, - )); - } - - Ok(AccountDetails { - account_header, - account_code, - vault_details, - storage_details: AccountStorageDetails { - header: storage_header, - map_details: storage_map_details, - }, - }) - } -} - -// HELPERS -// ================================================================================================ - -/// Expand [`AccountStorageRequest`] to a vector of slot requests. -fn expand_account_storage_request( - storage_request: AccountStorageRequest, - storage_header: &AccountStorageHeader, -) -> Vec { - match storage_request { - AccountStorageRequest::None => Vec::new(), - AccountStorageRequest::Explicit(requests) => requests, - AccountStorageRequest::AllStorageMaps => storage_header - .slots() - .filter(|slot| slot.slot_type() == StorageSlotType::Map) - .map(|slot| StorageMapRequest { - slot_name: slot.name().clone(), - slot_data: SlotData::All, - }) - .collect(), - } -} // This is intentionally conservative. Storage slot names can be up to u8::MAX bytes, and a // `limit_exceeded` map detail stores only the slot name plus the `too_many_entries` flag. @@ -407,7 +27,7 @@ const STORAGE_MAP_LIMIT_EXCEEDED_FIELD_MAX_LEN: usize = 263; // A conservative limit that makes sure that limit exceeded messages can be appended for all slots // in the response. -const MAX_ALL_STORAGE_MAPS_RESPONSE_PAYLOAD_WITH_BUDGET_RESERVED_FOR_LIMIT_EXCEEDED_SLOTS: usize = +pub(super) const MAX_ALL_STORAGE_MAPS_RESPONSE_PAYLOAD_WITH_BUDGET_RESERVED_FOR_LIMIT_EXCEEDED_SLOTS: usize = MAX_RESPONSE_PAYLOAD_BYTES - 256 * STORAGE_MAP_LIMIT_EXCEEDED_FIELD_MAX_LEN - 8192; // Conservative max length for storage map entries: key-value pairs, each one is four `fixed64` @@ -447,7 +67,7 @@ fn estimate_storage_map_details_field_len(details: &AccountStorageMapDetails) -> /// We reserve space for the "limit exceeded" responses in advance so we're safe to start appending /// "limit exceeded" at any point during iteration. #[expect(clippy::too_many_arguments)] -fn apply_all_storage_maps_response_budget( +pub(super) fn apply_all_storage_maps_response_budget( block_num: BlockNumber, witness: &AccountWitness, account_header: AccountHeader, @@ -504,7 +124,6 @@ fn apply_all_storage_maps_response_budget( }, } } - // TESTS // ================================================================================================ @@ -536,7 +155,8 @@ mod tests { use miden_protocol::testing::account_id::AccountIdBuilder; use miden_protocol::{EMPTY_WORD, Felt, Word}; - use super::{apply_all_storage_maps_response_budget, expand_account_storage_request}; + use super::super::expand_account_storage_request; + use super::apply_all_storage_maps_response_budget; fn storage_header() -> AccountStorageHeader { AccountStorageHeader::new(vec![ diff --git a/crates/store/src/state/view/batch_inputs.rs b/crates/store/src/state/view/batch_inputs.rs new file mode 100644 index 0000000000..237cd5cc47 --- /dev/null +++ b/crates/store/src/state/view/batch_inputs.rs @@ -0,0 +1,134 @@ +//! Batch input query for the block producer. +//! +//! Combines in-memory snapshot data (partial MMR) with database lookups, scoping the latter by +//! the view's tip so both sources describe the same block height. + +use std::collections::BTreeSet; + +use miden_node_proto::domain::batch::BatchInputs; +use miden_protocol::Word; +use miden_protocol::block::BlockNumber; +use miden_protocol::transaction::PartialBlockchain; + +use super::StateView; +use crate::errors::GetBatchInputsError; + +impl StateView { + /// Fetches the inputs for a transaction batch from the database. + /// + /// ## Inputs + /// + /// The function takes as input: + /// - The tx reference blocks are the set of blocks referenced by transactions in the batch. + /// - The unauthenticated note commitments are the set of commitments of unauthenticated notes + /// consumed by all transactions in the batch. For these notes, we attempt to find inclusion + /// proofs. Not all notes will exist in the DB necessarily, as some notes can be created and + /// consumed within the same batch. + /// + /// ## Outputs + /// + /// The function will return: + /// - A block inclusion proof for all tx reference blocks and for all blocks which are + /// referenced by a note inclusion proof. + /// - Note inclusion proofs for all notes that were found in the DB. + /// - The block header that the batch should reference, i.e. the latest known block. + pub async fn get_batch_inputs( + &self, + tx_reference_blocks: BTreeSet, + unauthenticated_note_commitments: BTreeSet, + ) -> Result { + if tx_reference_blocks.is_empty() { + return Err(GetBatchInputsError::TransactionBlockReferencesEmpty); + } + + let latest_block_num = self.tip(); + + // First we grab note inclusion proofs for the known notes. These proofs only prove that the + // note was included in a given block. We then also need to prove that each of those blocks + // is included in the chain. The proofs are scoped by the view's tip, so the database cannot + // report a note from a block the pinned snapshot cannot prove yet. + let note_proofs = self + .db + .select_note_inclusion_proofs(unauthenticated_note_commitments, latest_block_num) + .await + .map_err(GetBatchInputsError::SelectNoteInclusionProofError)?; + + // The set of blocks that the notes are included in. + let note_blocks = note_proofs.values().map(|proof| proof.location().block_num()); + + // Collect all blocks we need to query without duplicates, which is: + // - all blocks for which we need to prove note inclusion. + // - all blocks referenced by transactions in the batch. + let mut blocks: BTreeSet = tx_reference_blocks; + blocks.extend(note_blocks); + + // Remove the latest block from the to-be-tracked blocks as it will be the reference block + // for the batch itself and thus added to the MMR within the batch kernel, so there is no + // need to prove its inclusion. + blocks.remove(&*latest_block_num); + + // Scoping the blocks doubles as the validation that none lies beyond the view's tip. Scoped + // in descending order, so the first failure carries the highest block number. + let scoped_blocks = blocks + .iter() + .rev() + .map(|&block| { + self.scope_block(block).ok_or( + GetBatchInputsError::UnknownTransactionBlockReference { + highest_block_num: block, + latest_block_num: *latest_block_num, + }, + ) + }) + .collect::, _>>()?; + + // SAFETY: + // - The latest block num was retrieved from the view's blockchain from which we will + // also retrieve the proofs, so it is guaranteed to exist in that chain. + // - Scoping above proved that no block in the set is greater than the latest block number + // *and* the latest block num was removed from the set. Therefore only block numbers + // smaller than latest block num remain in the set. Therefore all the block numbers are + // guaranteed to exist in the chain state at latest block num. + let partial_mmr = + self.blockchain().partial_mmr_from_blocks(&blocks, *latest_block_num).expect( + "latest block num should exist and all blocks in set should be < than latest block", + ); + + // Fetch the reference block of the batch as part of this query, so we can avoid looking it + // up in a separate DB access. + let mut headers = self + .db + .select_block_headers( + scoped_blocks.into_iter().chain(std::iter::once(latest_block_num)), + ) + .await + .map_err(GetBatchInputsError::SelectBlockHeaderError)?; + + // Find and remove the batch reference block as we don't want to add it to the chain MMR. + let header_index = headers + .iter() + .enumerate() + .find_map(|(index, header)| (header.block_num() == *latest_block_num).then_some(index)) + .expect("DB should have returned the header of the batch reference block"); + + // The order doesn't matter for PartialBlockchain::new, so swap remove is fine. + let batch_reference_block_header = headers.swap_remove(header_index); + + // SAFETY: This should not error because: + // - we're passing exactly the block headers that we've added to the partial MMR, + // - so none of the block headers block numbers should exceed the chain length of the + // partial MMR, + // - and we've added blocks to a BTreeSet, so there can be no duplicates. + // + // We construct headers and partial MMR in concert, so they are consistent. This is why we + // can call the unchecked constructor. + let partial_block_chain = PartialBlockchain::new_unchecked(partial_mmr, headers) + .expect("partial mmr and block headers should be consistent"); + + Ok(BatchInputs { + batch_reference_block_header, + note_proofs, + partial_block_chain, + }) + } +} diff --git a/crates/store/src/state/view/block.rs b/crates/store/src/state/view/block.rs new file mode 100644 index 0000000000..adf40d6ca2 --- /dev/null +++ b/crates/store/src/state/view/block.rs @@ -0,0 +1,47 @@ +//! Block header reads. + +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::block::{BlockHeader, BlockNumber}; +use miden_protocol::crypto::merkle::mmr::MmrProof; + +use super::StateView; +use crate::COMPONENT; +use crate::errors::GetBlockHeaderError; + +impl StateView { + /// Queries a [BlockHeader] from the database, and returns it alongside its inclusion proof. + /// + /// If [None] is given as the value of `block_num`, the data for the latest [BlockHeader] is + /// returned. Returns `(None, None)` for blocks beyond this view's chain tip. + #[miden_instrument( + level = "debug", + target = COMPONENT, + err, + )] + pub async fn get_block_header( + &self, + block_num: Option, + include_mmr_proof: bool, + ) -> Result<(Option, Option), GetBlockHeaderError> { + // Resolve "latest" against the view's snapshot rather than the DB: mid-apply, the DB may + // already contain a block that the snapshot's blockchain cannot prove yet. Scoping the DB + // query by the view's tip keeps the header and MMR proof consistent. + let block_num = block_num.unwrap_or_else(|| *self.tip()); + let Some(scoped_block) = self.scope_block(block_num) else { + return Ok((None, None)); + }; + + let block_header = self.db.select_block_header_by_block_num(Some(scoped_block)).await?; + if let Some(header) = block_header { + let mmr_proof = if include_mmr_proof { + let mmr_proof = self.blockchain().open(header.block_num())?; + Some(mmr_proof) + } else { + None + }; + Ok((Some(header), mmr_proof)) + } else { + Ok((None, None)) + } + } +} diff --git a/crates/store/src/state/view/block_inputs.rs b/crates/store/src/state/view/block_inputs.rs new file mode 100644 index 0000000000..d9fb88ba54 --- /dev/null +++ b/crates/store/src/state/view/block_inputs.rs @@ -0,0 +1,168 @@ +//! Block input query for the block producer. +//! +//! Combines in-memory snapshot data (tree witnesses, partial MMR) with database lookups, scoping +//! the latter by the view's tip so both sources describe the same block height. + +use std::collections::{BTreeMap, BTreeSet}; + +use miden_protocol::Word; +use miden_protocol::account::AccountId; +use miden_protocol::block::account_tree::AccountWitness; +use miden_protocol::block::nullifier_tree::NullifierWitness; +use miden_protocol::block::{BlockInputs, BlockNumber}; +use miden_protocol::crypto::merkle::mmr::PartialMmr; +use miden_protocol::note::Nullifier; +use miden_protocol::transaction::PartialBlockchain; + +use super::{ScopedBlockNum, StateView}; +use crate::errors::GetBlockInputsError; + +type BlockInputWitnesses = ( + ScopedBlockNum, + Vec, + BTreeMap, + BTreeMap, + PartialMmr, +); + +impl StateView { + /// Returns data needed by the block producer to construct and prove the next block. + pub async fn get_block_inputs( + &self, + account_ids: Vec, + nullifiers: Vec, + unauthenticated_note_commitments: BTreeSet, + reference_blocks: BTreeSet, + ) -> Result { + // Get the note inclusion proofs from the DB first: the reference blocks of the note proofs + // are needed below to fetch their authentication paths in the chain MMR. The proofs are + // scoped by the view's tip, so the database cannot report a note from a block the pinned + // snapshot cannot prove yet. + let unauthenticated_note_proofs = self + .db + .select_note_inclusion_proofs(unauthenticated_note_commitments, self.tip()) + .await + .map_err(GetBlockInputsError::SelectNoteInclusionProofError)?; + + // The set of blocks that the notes are included in. + let note_proof_reference_blocks = + unauthenticated_note_proofs.values().map(|proof| proof.location().block_num()); + + // Collect all blocks we need to prove inclusion for, without duplicates. + let mut blocks = reference_blocks; + blocks.extend(note_proof_reference_blocks); + + let (latest, blocks, account_witnesses, nullifier_witnesses, partial_mmr) = + self.get_block_inputs_witnesses(blocks, &account_ids, &nullifiers)?; + + // Fetch the block headers for all blocks in the partial MMR plus the latest one which will + // be used as the previous block header of the block being built. + let mut headers = self + .db + .select_block_headers(blocks.into_iter().chain(std::iter::once(latest))) + .await + .map_err(GetBlockInputsError::SelectBlockHeaderError)?; + + // Find and remove the latest block as we must not add it to the chain MMR, since it is not + // yet in the chain. + let latest_block_header_index = headers + .iter() + .enumerate() + .find_map(|(index, header)| (header.block_num() == *latest).then_some(index)) + .expect("DB should have returned the header of the latest block header"); + + // The order doesn't matter for PartialBlockchain::new, so swap remove is fine. + let latest_block_header = headers.swap_remove(latest_block_header_index); + + // SAFETY: This should not error because: + // - we're passing exactly the block headers that we've added to the partial MMR, + // - so none of the block header's block numbers should exceed the chain length of the + // partial MMR, + // - and we've added blocks to a BTreeSet, so there can be no duplicates. + // + // We construct headers and partial MMR in concert, so they are consistent. This is why we + // can call the unchecked constructor. + let partial_block_chain = PartialBlockchain::new_unchecked(partial_mmr, headers) + .expect("partial mmr and block headers should be consistent"); + + Ok(BlockInputs::new( + latest_block_header, + partial_block_chain, + account_witnesses, + nullifier_witnesses, + unauthenticated_note_proofs, + )) + } + + /// Get account and nullifier witnesses for the requested account IDs and nullifiers, the + /// [`PartialMmr`] for the given blocks, and the blocks as scoped block numbers alongside the + /// scoped latest block. The MMR and the returned blocks won't contain the latest block. + fn get_block_inputs_witnesses( + &self, + mut blocks: BTreeSet, + account_ids: &[AccountId], + nullifiers: &[Nullifier], + ) -> Result { + self.with_inner_read_blocking(|inner| { + let latest_block_number = inner.latest_block_num(); + + // The latest block is not yet in the chain MMR, so we can't (and don't need to) prove + // its inclusion in the chain. + blocks.remove(&latest_block_number); + + // Scoping the blocks doubles as the validation that none lies beyond the view's tip + // (which equals the latest block number of this pinned snapshot). Scoped in descending + // order, so the first failure carries the highest block number. + let scoped_blocks = blocks + .iter() + .rev() + .map(|&block| { + self.scope_block(block).ok_or( + GetBlockInputsError::UnknownBatchBlockReference { + highest_block_number: block, + latest_block_number, + }, + ) + }) + .collect::, _>>()?; + + // Fetch the partial MMR at the state of the latest block with authentication paths for + // the provided set of blocks. + // + // SAFETY: + // - The latest block num was retrieved from the inner blockchain from which we will + // also retrieve the proofs, so it is guaranteed to exist in that chain. + // - Scoping above proved that no block in the set is greater than the latest block + // number *and* the latest block num was removed from the set. Therefore only block + // numbers smaller than latest block num remain in the set. Therefore all the block + // numbers are guaranteed to exist in the chain state at latest block num. + let partial_mmr = + inner.blockchain.partial_mmr_from_blocks(&blocks, latest_block_number).expect( + "latest block num should exist and all blocks in set should be < than latest block", + ); + + // Fetch witnesses for all accounts. + let account_witnesses = account_ids + .iter() + .copied() + .map(|account_id| (account_id, inner.account_tree.open_latest(account_id))) + .collect::>(); + + // Fetch witnesses for all nullifiers. We don't check whether the nullifiers are spent + // or not as this is done as part of proposing the block. + let nullifier_witnesses: BTreeMap = nullifiers + .iter() + .copied() + .map(|nullifier| (nullifier, inner.nullifier_tree.open(&nullifier))) + .collect(); + + Ok(( + self.tip(), + scoped_blocks, + account_witnesses, + nullifier_witnesses, + partial_mmr, + )) + }) + } +} diff --git a/crates/store/src/state/view/mod.rs b/crates/store/src/state/view/mod.rs new file mode 100644 index 0000000000..34379c751d --- /dev/null +++ b/crates/store/src/state/view/mod.rs @@ -0,0 +1,165 @@ +//! Request-scoped, consistent read view of the store. +//! +//! All store reads go through [`StateView`]: it pins one state snapshot for its whole lifetime, +//! and every block-scoped database query it exposes is bounded by that snapshot's height (via the +//! [`scoped`] proof types). This makes it impossible to implement a read whose tree and database +//! halves observe different chain tips — mid-apply, the database may already contain rows for a +//! block the snapshot cannot prove yet. The only deliberately unscoped reads are the +//! content-addressed note lookups and the network-account classification. +//! +//! The submodules hold the read endpoints, all `impl StateView`; the snapshot internals +//! ([`StateSnapshot`]) are only visible within this module tree, so no other part of the store +//! can reach the trees directly. + +use std::ops::RangeInclusive; +use std::sync::Arc; + +use miden_protocol::block::{BlockNumber, Blockchain}; +use tracing::Span; + +use crate::account_state_forest::{AccountStateForest, AccountStateForestBackendReader}; +use crate::db::Db; +use crate::errors::RangeBeyondTip; +use crate::state::State; + +mod scoped; +pub use scoped::{ScopedBlockNum, ScopedBlockRange}; + +mod snapshot; +pub(in crate::state) use snapshot::{ + PublishedGenerations, + SNAPSHOTS_LIVE_WARN_THRESHOLD, + SnapshotGuard, + StateSnapshot, +}; + +mod account; +mod batch_inputs; +mod block; +mod block_inputs; +mod note; +mod sync; + +mod transaction_inputs; +pub use transaction_inputs::TransactionInputs; + +// STATE VIEW +// ================================================================================================ + +/// A consistent read view of the store, pinned at its snapshot's block height. +/// +/// Obtained from [`State::view`]; create one per request and drop it when the request completes. +/// Holding a view pins a snapshot generation (and thereby the `RocksDB` snapshots backing the +/// trees), so it must not be stored in long-lived structs; leaked or slow readers are reported by +/// the store's snapshot-lifetime warnings. +/// +/// Reads that are technically not block-scoped (e.g. content-addressed note scripts) also live +/// here so that every read path flows through a single, consistently-scoped type. +pub struct StateView { + snapshot: Arc, + db: Arc, +} + +impl State { + /// Returns a read view pinned at the current chain tip (wait-free, no lock required). + /// + /// The view is frozen: it is unaffected if the writer publishes a new snapshot while it is + /// held. + /// + /// Use this only for one single-expression read (`state.view().get_account(..)`), + /// where the temporary view drops at the end of the statement. Two `view()` calls observe + /// potentially *different* snapshots — a block can commit between them — so reads that must + /// be mutually consistent (e.g. a query and the tip it was served at) must share one view via + /// [`Self::with_view`]. Binding a view to a variable is also discouraged: it keeps the + /// snapshot generation pinned until the end of the scope. + pub fn view(&self) -> StateView { + StateView { + snapshot: self.latest_snapshot.load_full(), + db: Arc::clone(&self.db), + } + } + + /// Runs a read operation over a view pinned at the current chain tip, dropping the view — and + /// releasing its snapshot generation — as soon as the operation completes. + /// + /// This is the required form whenever multiple reads must observe the *same* snapshot: the + /// closure's view is one consistent generation, whereas consecutive [`Self::view`] calls may + /// straddle a commit. The typical case is pairing a query with [`StateView::tip`] so a + /// response reports exactly the height it was served at. + /// + /// The closure receives a shared reference, so the view cannot escape the call, and the + /// snapshot is not pinned through unrelated work that follows (e.g. response encoding). A + /// *single* read doesn't need it — `state.view().get_account(..)` drops its temporary view at + /// the end of the statement. + /// + /// Work in the closure should be kept to low-complexity compute over the view, ideally with no + /// I/O and no other `.await` points. Anything slower holds the pinned snapshot, and therefore + /// its underlying `RocksDB` snapshot, for as long as it runs. The snapshot's lifetime is logged + /// as a warning if held too long, but that is a backstop, not a substitute for keeping closures + /// short. + pub async fn with_view(&self, f: impl AsyncFnOnce(&StateView) -> R) -> R { + let view = self.view(); + f(&view).await + } +} + +impl StateView { + /// Returns this view's tip as a scoped block number for tip-bounded database queries. + pub fn tip(&self) -> ScopedBlockNum { + ScopedBlockNum::new(self.snapshot.latest_block_num()) + } + + /// Returns the pinned snapshot's blockchain MMR. + /// + /// The MMR is the only part of the snapshot that is purely in-memory and therefore safe to + /// access directly on an async worker thread. The account and nullifier trees may be backed + /// by `RocksDB` and are deliberately not reachable here — they must be accessed through + /// [`Self::with_inner_read_blocking`]. + fn blockchain(&self) -> &Blockchain { + &self.snapshot.blockchain + } + + /// Validates that `block_num` does not exceed this view's chain tip, returning the scoped block + /// number required by block-bounded database queries. + fn scope_block(&self, block_num: BlockNumber) -> Option { + (block_num <= *self.tip()).then(|| ScopedBlockNum::new(block_num)) + } + + /// Validates that `range` does not extend beyond this view's chain tip, returning the scoped + /// range required by range-bounded database queries. + /// + /// Every range-scoped read on this type calls this before touching the database, so callers + /// never need to pre-validate ranges themselves. + fn scope_range( + &self, + range: RangeInclusive, + ) -> Result { + let tip = *self.tip(); + if *range.end() > tip { + return Err(RangeBeyondTip { chain_tip: tip, block_to: *range.end() }); + } + Ok(ScopedBlockRange::new(range)) + } + + /// Runs a synchronous read-only operation over the pinned state snapshot on Tokio's blocking + /// path. + /// + /// The account and nullifier trees may be backed by `RocksDB`, so tree access must not run on + /// an async worker thread directly. This helper preserves the current tracing span while + /// moving the closure body into `block_in_place`. + fn with_inner_read_blocking(&self, f: impl FnOnce(&StateSnapshot) -> R) -> R { + let span = Span::current(); + tokio::task::block_in_place(|| span.in_scope(|| f(&self.snapshot))) + } + + /// Runs a synchronous read-only operation over the account state forest snapshot on Tokio's + /// blocking path. + /// + /// See [`Self::with_inner_read_blocking`] for why this uses `block_in_place`. + fn with_forest_read_blocking( + &self, + f: impl FnOnce(&AccountStateForest) -> R, + ) -> R { + self.with_inner_read_blocking(|snapshot| f(&snapshot.forest)) + } +} diff --git a/crates/store/src/state/view/note.rs b/crates/store/src/state/view/note.rs new file mode 100644 index 0000000000..98ddf22795 --- /dev/null +++ b/crates/store/src/state/view/note.rs @@ -0,0 +1,33 @@ +//! Note reads. +//! +//! These are content-addressed lookups and technically not block-scoped, but they live on +//! [`StateView`] so that every read path flows through the same type. + +use miden_protocol::Word; +use miden_protocol::note::{NoteId, NoteScript}; + +use super::StateView; +use crate::db::NoteRecord; +use crate::errors::DatabaseError; + +impl StateView { + /// Queries a list of notes from the database. + /// + /// If the provided list of [`NoteId`]s is empty or no note matches, an empty list is + /// returned. This lookup is deliberately not bounded by this view's tip (latest-wins): a note + /// committed while a block is being applied may be returned before the snapshot advances. + pub async fn get_notes_by_id( + &self, + note_ids: Vec, + ) -> Result, DatabaseError> { + self.db.select_notes_by_id(note_ids).await + } + + /// Returns the script for a note by its root. + pub async fn get_note_script_by_root( + &self, + root: Word, + ) -> Result, DatabaseError> { + self.db.select_note_script_by_root(root).await + } +} diff --git a/crates/store/src/state/view/scoped.rs b/crates/store/src/state/view/scoped.rs new file mode 100644 index 0000000000..0691febb9b --- /dev/null +++ b/crates/store/src/state/view/scoped.rs @@ -0,0 +1,87 @@ +//! Proof-of-validation block-number types issued by [`StateView`](super::StateView). +//! +//! Tip-scoped database queries take these types instead of raw block numbers, so a query whose +//! bound was not validated against a state view's tip is not expressible: the only constructors +//! live on [`StateView`](super::StateView), which checks the bound against its pinned snapshot. +//! +//! Chain tips are monotonic, so a value issued by an older view remains valid for any later +//! state — holding one across view lifetimes is sound, if pointless. + +use std::ops::{Deref, RangeInclusive}; + +use miden_protocol::block::BlockNumber; + +/// A block number proven to be at or below the issuing view's chain tip. +#[derive(Debug, Clone, Copy)] +pub struct ScopedBlockNum(BlockNumber); + +impl ScopedBlockNum { + /// Issued by [`StateView`](super::StateView) after validating the bound against its tip. + pub(super) fn new(block_num: BlockNumber) -> Self { + Self(block_num) + } + + /// Constructs a scoped block number without validation. + /// + /// Test-only: lets database tests exercise scoped queries without a running state. + #[cfg(test)] + pub(crate) fn new_unchecked(block_num: BlockNumber) -> Self { + Self(block_num) + } + + /// Derives a scoped range ending at this validated block number. + /// + /// Sound for any `start` because only a range's upper bound carries the proof obligation. + /// A `start` beyond this block number would however produce an empty range, which the range + /// queries reject as an invalid block range. Used for paginating over a validated bound in + /// sub-ranges. + pub(crate) fn range_from(self, start: BlockNumber) -> ScopedBlockRange { + debug_assert!(start <= self.0, "derived range start {start} exceeds its end {}", self.0); + ScopedBlockRange(start..=self.0) + } +} + +/// The validated block number is read by dereferencing (`*scoped`); [`DerefMut`] is deliberately +/// not implemented, as mutating the inner value would invalidate the proof. +/// +/// [`DerefMut`]: std::ops::DerefMut +impl Deref for ScopedBlockNum { + type Target = BlockNumber; + + fn deref(&self) -> &BlockNumber { + &self.0 + } +} + +/// A block range whose upper bound is proven to be at or below the issuing view's chain tip. +#[derive(Debug, Clone)] +pub struct ScopedBlockRange(RangeInclusive); + +impl ScopedBlockRange { + /// Issued by [`StateView`](super::StateView) after validating the range against its tip. + pub(super) fn new(range: RangeInclusive) -> Self { + Self(range) + } + + /// Returns the start of the validated range. + pub(crate) fn start(&self) -> BlockNumber { + *self.0.start() + } + + /// Returns the end of the validated range. + pub(crate) fn end(&self) -> BlockNumber { + *self.0.end() + } + + /// Returns the end of the validated range as a scoped block number. + /// + /// Sound because the range's upper bound is exactly what its proof covers. + pub(crate) fn scoped_end(&self) -> ScopedBlockNum { + ScopedBlockNum(*self.0.end()) + } + + /// Returns the validated range. + pub(crate) fn into_inner(self) -> RangeInclusive { + self.0 + } +} diff --git a/crates/store/src/state/view/snapshot.rs b/crates/store/src/state/view/snapshot.rs new file mode 100644 index 0000000000..50ecd3f921 --- /dev/null +++ b/crates/store/src/state/view/snapshot.rs @@ -0,0 +1,287 @@ +//! In-memory snapshot machinery for lock-free reads. +//! +//! Readers access the store's tree state through immutable [`StateSnapshot`] snapshots published +//! by the block writer after each committed block. [`SnapshotGuard`] tracks how many snapshot +//! generations are pinned by readers, since each generation pins a `RocksDB` snapshot. The writer +//! additionally remembers each published generation in [`PublishedGenerations`], whose oldest +//! still-pinned height feeds snapshot-aware history pruning, since SQLite reads have no +//! point-in-time protection equivalent to the `RocksDB` snapshots backing the trees. + +use std::collections::VecDeque; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, OnceLock, Weak}; +use std::time::{Duration, Instant}; + +use miden_protocol::block::nullifier_tree::NullifierTree; +use miden_protocol::block::{BlockNumber, Blockchain}; +use miden_protocol::crypto::merkle::smt::LargeSmt; + +use crate::COMPONENT; +use crate::account_state_forest::{ + AccountStateForest, + AccountStateForestBackendReader, + HISTORICAL_BLOCK_RETENTION, +}; +use crate::accounts::AccountTreeWithHistory; +use crate::state::loader::TreeStorageReader; + +/// Time held past supersession above which [`SnapshotGuard`] logs a warning on release. +/// +/// Readers are expected to be request-scoped, so a superseded generation outliving several block +/// intervals indicates a slow or leaked reader pinning a `RocksDB` snapshot (see +/// [`SnapshotGuard`]). The clock starts at supersession rather than creation: the latest +/// generation is always pinned by the published pointer, so time spent as the current generation +/// (idle chains, shutdown) says nothing about reader behaviour. +const SNAPSHOT_SUPERSEDED_WARN_THRESHOLD: Duration = Duration::from_secs(10); + +/// Number of live snapshot generations above which the block writer logs a warning after +/// publishing a new snapshot. +/// +/// Steady state is 1-2 generations: the just-published snapshot plus predecessors briefly pinned +/// by in-flight requests. A sustained higher count means slow or leaked readers are holding old +/// generations alive (see [`SnapshotGuard`]). +pub(in crate::state) const SNAPSHOTS_LIVE_WARN_THRESHOLD: u64 = 4; + +/// Upper bound on how far the snapshot-aware pruning tip may lag the chain tip. +/// +/// History pruning keys off the oldest live snapshot generation (see +/// [`PublishedGenerations::prune_tip`]), so a leaked or pathologically slow reader would +/// otherwise stall pruning indefinitely. Beyond this +/// many blocks of lag the writer prunes anyway, accepting the historical-read race for that reader +/// (which the snapshot-lifetime warnings have long since reported). One full retention window, so +/// worst-case retained history is bounded at twice the window. +const SNAPSHOT_PRUNE_LAG_CAP: u32 = HISTORICAL_BLOCK_RETENTION; + +// PUBLISHED GENERATIONS +// ================================================================================================ + +/// The writer's log of published snapshot generations, used to derive the oldest height still +/// pinned by a reader. +/// +/// Owned exclusively by the writer — no locks or shared state. Liveness is not tracked +/// separately: a [`Weak`] per generation asks the snapshot's own [`Arc`] refcount, which is the +/// ground truth for "some reader can still see this height". Dead and no-longer-relevant entries +/// are discarded on each [`Self::prune_tip`] call (once per applied block), which bounds the +/// deque to roughly [`SNAPSHOT_PRUNE_LAG_CAP`] entries even when a reader leaks its snapshot. +/// +/// Generic over the pinned type for testability; the writer uses `T = StateSnapshot`. +pub(in crate::state) struct PublishedGenerations { + /// Published generations in ascending height order. + entries: VecDeque<(BlockNumber, Weak)>, +} + +impl PublishedGenerations { + pub(in crate::state) fn new() -> Self { + Self { entries: VecDeque::new() } + } + + /// Records a newly published generation. Heights must be recorded in ascending order. + pub(in crate::state) fn record(&mut self, height: BlockNumber, pinned: &Arc) { + debug_assert!( + self.entries.back().is_none_or(|(back, _)| *back < height), + "generation {height} published out of order", + ); + self.entries.push_back((height, Arc::downgrade(pinned))); + } + + /// Returns the effective chain tip for history pruning. + /// + /// The store's SQLite reads are scoped only by an upper block bound, with no point-in-time + /// protection equivalent to the `RocksDB` snapshots backing the trees. Pruning therefore + /// treats the oldest still-pinned generation as the tip: a generation pinned at height `H` + /// keeps the same retention window it had when `H` was the tip, and pruning simply lags until + /// it is released. The lag is capped at [`SNAPSHOT_PRUNE_LAG_CAP`] blocks so a leaked reader + /// cannot stall pruning indefinitely: entries below the cap's floor are discarded despite + /// still being pinned, as are entries that are no longer pinned. + pub(in crate::state) fn prune_tip(&mut self, chain_tip: BlockNumber) -> BlockNumber { + let lag_floor = chain_tip.as_u32().saturating_sub(SNAPSHOT_PRUNE_LAG_CAP); + while let Some((height, pinned)) = self.entries.front() { + // Drop entries below the lag floor or that are no longer pinned. + if height.as_u32() < lag_floor || pinned.strong_count() == 0 { + self.entries.pop_front(); + } else { + break; + } + } + // Return the prune tip, which is the oldest pinned generation or the chain tip. + self.entries.front().map_or(chain_tip, |(height, _)| (*height).min(chain_tip)) + } +} + +// SNAPSHOT GUARD +// ================================================================================================ + +/// RAII member of [`StateSnapshot`] that tracks the number of live snapshot generations. +/// +/// [`StateSnapshot`] is dropped exactly when the last [`Arc`] reference to it is released, so the +/// shared counter reports how many distinct snapshot generations are currently pinned by readers. +/// A sustained count above 1-2 means slow readers are holding old generations alive. Each +/// generation pins a `RocksDB` snapshot, which delays garbage collection of superseded key +/// versions during compaction (compaction itself keeps running); the retained garbage grows with +/// write churn for as long as the snapshot is held and is reclaimed once it is released. A held +/// generation also holds back SQLite history pruning (see [`PublishedGenerations::prune_tip`]). +/// +/// Readers are expected to be request-scoped, so a superseded generation should be released well +/// within a block interval. Outliving supersession by more than +/// [`SNAPSHOT_SUPERSEDED_WARN_THRESHOLD`] is logged at warn level; a generation that is never +/// superseded (the latest at shutdown) is released silently regardless of age. +pub(in crate::state) struct SnapshotGuard { + live: Arc, + created_at: Instant, + /// Set by the writer when a newer generation replaces this one as the published snapshot. + superseded_at: OnceLock, + block_num: BlockNumber, +} + +impl SnapshotGuard { + pub(in crate::state) fn new(live: Arc, block_num: BlockNumber) -> Self { + live.fetch_add(1, Ordering::Relaxed); + Self { + live, + created_at: Instant::now(), + superseded_at: OnceLock::new(), + block_num, + } + } + + /// Marks this generation as superseded by a newer published snapshot, starting the clock + /// against which slow readers are measured. Only the first call takes effect. + pub(in crate::state) fn mark_superseded(&self) { + let _ = self.superseded_at.set(Instant::now()); + } +} + +impl Drop for SnapshotGuard { + fn drop(&mut self) { + let remaining = self.live.fetch_sub(1, Ordering::Relaxed) - 1; + let lifetime_ms = u64::try_from(self.created_at.elapsed().as_millis()).unwrap_or(u64::MAX); + let block_num = self.block_num.as_u32(); + let superseded_for = self.superseded_at.get().map(Instant::elapsed); + if let Some(superseded_for) = + superseded_for.filter(|held| *held > SNAPSHOT_SUPERSEDED_WARN_THRESHOLD) + { + let superseded_for_ms = u64::try_from(superseded_for.as_millis()).unwrap_or(u64::MAX); + tracing::warn!( + target: COMPONENT, + block_num, + snapshot.lifetime_ms = lifetime_ms, + snapshot.superseded_for_ms = superseded_for_ms, + snapshots.live = remaining, + "state snapshot held for excessive time after supersession", + ); + } else { + tracing::debug!( + target: COMPONENT, + block_num, + snapshot.lifetime_ms = lifetime_ms, + snapshots.live = remaining, + "state snapshot released", + ); + } + } +} + +// STATE SNAPSHOT +// ================================================================================================ + +/// Immutable snapshot of the store's tree state published after each committed block. +/// +/// The trees are backed by read-only snapshot storage ([`TreeStorageReader`] / +/// [`AccountStateForestBackendReader`]), so any number of readers can access the data concurrently +/// without holding a lock and without blocking the writer. +/// +/// The writer and lifecycle can *construct* snapshots via [`Self::new`], but the fields are only +/// readable within the view module tree: every read outside it must go through +/// [`StateView`](super::StateView). +pub(in crate::state) struct StateSnapshot { + pub(super) nullifier_tree: NullifierTree>, + pub(super) blockchain: Blockchain, + pub(super) account_tree: AccountTreeWithHistory, + pub(super) forest: AccountStateForest, + /// Keeps the live-snapshot count accurate; see [`SnapshotGuard`]. + guard: SnapshotGuard, +} + +impl StateSnapshot { + /// Assembles a snapshot from reader views of the trees and the guard tracking its generation. + pub(in crate::state) fn new( + nullifier_tree: NullifierTree>, + blockchain: Blockchain, + account_tree: AccountTreeWithHistory, + forest: AccountStateForest, + guard: SnapshotGuard, + ) -> Self { + Self { + nullifier_tree, + blockchain, + account_tree, + forest, + guard, + } + } + + /// Marks this snapshot as superseded by a newer published generation; see + /// [`SnapshotGuard::mark_superseded`]. + pub(in crate::state) fn mark_superseded(&self) { + self.guard.mark_superseded(); + } + + /// Returns the latest block number. + pub(in crate::state) fn latest_block_num(&self) -> BlockNumber { + self.blockchain + .chain_tip() + .expect("chain should always have at least the genesis block") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn prune_tip_tracks_oldest_pinned_height_across_out_of_order_drops() { + let mut published = PublishedGenerations::::new(); + let tip = BlockNumber::from(100); + + // No live generations: prune at the tip. + assert_eq!(published.prune_tip(tip), tip); + + let gen_97 = Arc::new(97); + let gen_98 = Arc::new(98); + let gen_99 = Arc::new(99); + published.record(BlockNumber::from(97), &gen_97); + published.record(BlockNumber::from(98), &gen_98); + published.record(BlockNumber::from(99), &gen_99); + assert_eq!(published.prune_tip(tip), BlockNumber::from(97)); + + // Dropping a middle generation leaves the oldest unchanged. + drop(gen_98); + assert_eq!(published.prune_tip(tip), BlockNumber::from(97)); + + drop(gen_97); + assert_eq!(published.prune_tip(tip), BlockNumber::from(99)); + + // A pinned generation never advances pruning past the tip. + assert_eq!(published.prune_tip(BlockNumber::from(98)), BlockNumber::from(98)); + + drop(gen_99); + assert_eq!(published.prune_tip(tip), tip); + } + + #[test] + fn prune_tip_discards_leaked_entries_below_the_lag_floor() { + let mut published = PublishedGenerations::::new(); + let leaked = Arc::new(1); + published.record(BlockNumber::from(1), &leaked); + + // While the leaked generation is within the lag cap it holds pruning back; near genesis the + // lag floor saturates to zero. + let tip = BlockNumber::from(SNAPSHOT_PRUNE_LAG_CAP); + assert_eq!(published.prune_tip(tip), BlockNumber::from(1)); + + // Once the tip advances past the cap it is discarded despite still being pinned, and no + // longer holds pruning back. + let tip = BlockNumber::from(SNAPSHOT_PRUNE_LAG_CAP + 2); + assert_eq!(published.prune_tip(tip), tip); + } +} diff --git a/crates/store/src/state/sync_state.rs b/crates/store/src/state/view/sync.rs similarity index 71% rename from crates/store/src/state/sync_state.rs rename to crates/store/src/state/view/sync.rs index b871be6a3c..603c078beb 100644 --- a/crates/store/src/state/sync_state.rs +++ b/crates/store/src/state/view/sync.rs @@ -5,7 +5,7 @@ use miden_protocol::account::AccountId; use miden_protocol::block::{BlockHeader, BlockNumber, BlockSignatures}; use miden_protocol::crypto::merkle::mmr::{Forest, MmrDelta, MmrProof}; -use super::State; +use super::StateView; use crate::COMPONENT; use crate::db::models::queries::StorageMapValuesPage; use crate::db::{AccountVaultValue, NoteSyncUpdate, NullifierInfo}; @@ -14,18 +14,26 @@ use crate::errors::{DatabaseError, NoteSyncError, StateSyncError}; // STATE SYNCHRONIZATION ENDPOINTS // ================================================================================================ -impl State { +impl StateView { /// Returns the complete transaction records for the specified accounts within the specified /// block range, including state commitments and note IDs. + /// + /// Returns [`RangeBeyondTip`](crate::errors::RangeBeyondTip) if the range extends beyond this + /// view's chain tip. pub async fn sync_transactions( &self, account_ids: Vec, block_range: RangeInclusive, ) -> Result<(BlockNumber, Vec), DatabaseError> { + let block_range = self.scope_range(block_range)?; self.db.select_transactions_records(account_ids, block_range).await } - /// Returns the chain MMR delta and the `block_to` block header for the specified block range. + /// Returns the chain MMR delta and the block header at the range's end for the specified + /// block range. + /// + /// Returns [`RangeBeyondTip`](crate::errors::RangeBeyondTip) if the range extends beyond this + /// view's chain tip. #[miden_instrument( level = "debug", target = COMPONENT, @@ -35,16 +43,18 @@ impl State { &self, block_range: RangeInclusive, ) -> Result<(MmrDelta, BlockHeader, BlockSignatures), StateSyncError> { - let block_from = *block_range.start(); - let block_to = *block_range.end(); + let block_range = self.scope_range(block_range)?; + + let block_from = block_range.start(); + let block_to = block_range.end(); - // SAFETY: block_to has been validated to be <= the effective tip (chain tip or latest - // proven block) by the caller, so it must exist in the database. + // The scoped range's end is committed (at or below this view's tip), so its header must + // exist in the database. let (block_header, signatures) = self .db - .select_block_header_and_signatures_by_block_num(block_to) + .select_block_header_and_signatures_by_block_num(block_range.scoped_end()) .await? - .expect("block_to should exist in the database"); + .expect("the range-end header should exist in the database"); if block_from == block_to { return Ok(( @@ -70,10 +80,7 @@ impl State { let to_forest = block_to.as_usize(); let mmr_delta = self - .inner - .read() - .await - .blockchain + .blockchain() .as_mmr() .get_delta( Forest::new(from_forest).expect("from_forest fits in u32"), @@ -92,6 +99,9 @@ impl State { /// /// Also returns the last block number checked. If this equals `block_range.end()`, the /// sync is complete. + /// + /// Returns [`RangeBeyondTip`](crate::errors::RangeBeyondTip) if the range extends beyond this + /// view's chain tip. #[miden_instrument( level = "debug", target = COMPONENT, @@ -102,24 +112,22 @@ impl State { note_tags: Vec, block_range: RangeInclusive, ) -> Result<(Vec<(NoteSyncUpdate, MmrProof)>, BlockNumber), NoteSyncError> { - let block_end = *block_range.end(); + let block_range = self.scope_range(block_range)?; + + let block_end = block_range.end(); // The MMR at forest N contains proofs for blocks 0..N-1, so we use block_end + 1 to include - // the proof for block_end. SAFETY: it is ensured that block_end <= chain_tip, and the - // blockchain MMR always has at least chain_tip + 1 leaves. + // the proof for block_end. SAFETY: block_end <= this view's tip (checked above), and the + // view's blockchain MMR always has at least tip + 1 leaves. let mmr_checkpoint = block_end + 1; let note_syncs = self.db.get_note_sync_multi(block_range, note_tags.into()).await?; let mut results = Vec::new(); - { - let inner = self.inner.read().await; - - for note_sync in note_syncs { - let mmr_proof = - inner.blockchain.open_at(note_sync.block_header.block_num(), mmr_checkpoint)?; - results.push((note_sync, mmr_proof)); - } + for note_sync in note_syncs { + let mmr_proof = + self.blockchain().open_at(note_sync.block_header.block_num(), mmr_checkpoint)?; + results.push((note_sync, mmr_proof)); } // if results is empty, return `block_end` since the sync is complete. @@ -129,12 +137,17 @@ impl State { Ok((results, last_block_checked)) } + /// Returns nullifiers matching the given prefixes that were created within a block range. + /// + /// Returns [`RangeBeyondTip`](crate::errors::RangeBeyondTip) if the range extends beyond this + /// view's chain tip. pub async fn sync_nullifiers( &self, prefix_len: u32, nullifier_prefixes: Vec, block_range: RangeInclusive, ) -> Result<(Vec, BlockNumber), DatabaseError> { + let block_range = self.scope_range(block_range)?; self.db .select_nullifiers_by_prefix(prefix_len, nullifier_prefixes, block_range) .await @@ -144,20 +157,28 @@ impl State { // -------------------------------------------------------------------------------------------- /// Returns account vault updates for specified account within a block range. + /// + /// Returns [`RangeBeyondTip`](crate::errors::RangeBeyondTip) if the range extends beyond this + /// view's chain tip. pub async fn sync_account_vault( &self, account_id: AccountId, block_range: RangeInclusive, ) -> Result<(BlockNumber, Vec), DatabaseError> { + let block_range = self.scope_range(block_range)?; self.db.get_account_vault_sync(account_id, block_range).await } /// Returns storage map values for syncing within a block range. + /// + /// Returns [`RangeBeyondTip`](crate::errors::RangeBeyondTip) if the range extends beyond this + /// view's chain tip. pub async fn sync_account_storage_maps( &self, account_id: AccountId, block_range: RangeInclusive, ) -> Result { + let block_range = self.scope_range(block_range)?; self.db.select_storage_map_sync_values(account_id, block_range, None).await } } diff --git a/crates/store/src/state/view/transaction_inputs.rs b/crates/store/src/state/view/transaction_inputs.rs new file mode 100644 index 0000000000..fcab7391db --- /dev/null +++ b/crates/store/src/state/view/transaction_inputs.rs @@ -0,0 +1,92 @@ +//! Transaction input query for the block producer's transaction validation. + +use std::collections::HashSet; +use std::ops::ControlFlow; + +use miden_node_utils::formatting::format_array; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::Word; +use miden_protocol::account::AccountId; +use miden_protocol::note::Nullifier; + +use super::StateView; +use crate::COMPONENT; +use crate::db::NullifierInfo; +use crate::errors::DatabaseError; + +/// Store-level inputs for validating a proven transaction. +#[derive(Debug, Default)] +pub struct TransactionInputs { + pub account_commitment: Word, + pub nullifiers: Vec, + pub found_unauthenticated_notes: HashSet, + pub new_account_id_prefix_is_unique: Option, +} + +impl StateView { + /// Returns data needed by the block producer to verify transaction validity. + #[miden_instrument( + target = COMPONENT, + fields( + account.id=%account_id, + nullifiers = %format_array(nullifiers), + ), + )] + pub async fn get_transaction_inputs( + &self, + account_id: AccountId, + nullifiers: &[Nullifier], + unauthenticated_note_commitments: Vec, + ) -> Result { + let tree_inputs = self.with_inner_read_blocking(|inner| { + let account_commitment = inner.account_tree.get_latest_commitment(account_id); + + let new_account_id_prefix_is_unique = if account_commitment.is_empty() { + Some(!inner.account_tree.contains_account_id_prefix_in_latest(account_id.prefix())) + } else { + None + }; + + // Non-unique account Id prefixes for new accounts are not allowed, so the transaction + // cannot be valid and the response is already complete. + if let Some(false) = new_account_id_prefix_is_unique { + return ControlFlow::Break(TransactionInputs { + new_account_id_prefix_is_unique, + ..Default::default() + }); + } + + let nullifiers = nullifiers + .iter() + .map(|nullifier| NullifierInfo { + nullifier: *nullifier, + block_num: inner.nullifier_tree.get_block_num(nullifier).unwrap_or_default(), + }) + .collect(); + + ControlFlow::Continue((account_commitment, nullifiers, new_account_id_prefix_is_unique)) + }); + // `Break` carries a complete response (duplicate account ID prefix), so it is returned + // as-is without the note lookup below; `Continue` carries the tree reads needed to build + // the full response. + let (account_commitment, nullifiers, new_account_id_prefix_is_unique) = match tree_inputs { + ControlFlow::Continue(inputs) => inputs, + ControlFlow::Break(response) => return Ok(response), + }; + + // Scope the note lookup by the view's tip so the result is consistent with the tree reads + // above: mid-apply, the DB may already contain notes from a block the snapshot does not + // include yet. + let found_unauthenticated_notes = self + .db + .select_existing_note_commitments(unauthenticated_note_commitments, self.tip()) + .await?; + + Ok(TransactionInputs { + account_commitment, + nullifiers, + found_unauthenticated_notes, + new_account_id_prefix_is_unique, + }) + } +} diff --git a/crates/store/src/state/writer/apply_block.rs b/crates/store/src/state/writer/apply_block.rs new file mode 100644 index 0000000000..e751c5e3f7 --- /dev/null +++ b/crates/store/src/state/writer/apply_block.rs @@ -0,0 +1,53 @@ +use miden_node_proto::domain::proof_request::BlockProofRequest; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::batch::OrderedBatches; +use miden_protocol::block::{BlockInputs, BlockNumber, SignedBlock}; +use miden_protocol::utils::serde::Serializable; + +use crate::COMPONENT; +use crate::errors::ApplyBlockWithProvingInputsError; +use crate::state::BlockWriter; + +impl BlockWriter { + /// Saves proving inputs for a signed block and applies it to the state. + /// + /// Used by the in-process block producer after it has built and signed a block. + #[miden_instrument( + target = COMPONENT, + err, + )] + pub async fn apply_block_with_proving_inputs( + &mut self, + ordered_batches: OrderedBatches, + block_inputs: BlockInputs, + signed_block: SignedBlock, + ) -> Result<(), ApplyBlockWithProvingInputsError> { + let block_header = signed_block.header().clone(); + let block_num = block_header.block_num(); + + let proving_inputs = BlockProofRequest { + tx_batches: ordered_batches, + block_header, + block_inputs, + }; + + self.save_proving_inputs(block_num, &proving_inputs) + .await + .map_err(ApplyBlockWithProvingInputsError::SaveProvingInputs)?; + + self.apply_block(signed_block) + .await + .map_err(ApplyBlockWithProvingInputsError::ApplyBlock) + } + + /// Saves the proving inputs for the given block to the block store. + pub async fn save_proving_inputs( + &self, + block_num: BlockNumber, + proving_inputs: &BlockProofRequest, + ) -> std::io::Result<()> { + self.block_store + .save_proving_inputs(block_num, &proving_inputs.to_bytes()) + .await + } +} diff --git a/crates/store/src/state/apply_proof.rs b/crates/store/src/state/writer/apply_proof.rs similarity index 81% rename from crates/store/src/state/apply_proof.rs rename to crates/store/src/state/writer/apply_proof.rs index 1b20e7082c..029557415d 100644 --- a/crates/store/src/state/apply_proof.rs +++ b/crates/store/src/state/writer/apply_proof.rs @@ -4,9 +4,9 @@ use miden_protocol::block::{BlockNumber, BlockProof}; use miden_protocol::utils::serde::Deserializable; use crate::COMPONENT; -use crate::state::{Finality, ProofNotification, State}; +use crate::state::{ProofNotification, ProofWriter}; -impl State { +impl ProofWriter { /// Saves a block proof, advances the proven-in-sequence tip, and notifies replica subscribers. /// /// # Errors @@ -21,17 +21,17 @@ impl State { ), )] pub async fn apply_proof( - &self, + &mut self, block_num: BlockNumber, proof_bytes: Vec, ) -> anyhow::Result<()> { - let expected = self.proven_tip.read().child(); + let expected = self.state.proven_tip().child(); ensure!( block_num == expected, "out-of-sequence proof: expected block {expected}, got {block_num}", ); - let committed_tip = self.chain_tip(Finality::Committed).await; + let committed_tip = self.state.committed_tip(); ensure!( block_num <= committed_tip, "proof for uncommitted block {block_num} exceeds committed tip {committed_tip}", @@ -39,11 +39,12 @@ impl State { verify_block_proof(block_num, &proof_bytes)?; - self.block_store.commit_proof(block_num, &proof_bytes).await?; - self.proof_cache + self.state.block_store.commit_proof(block_num, &proof_bytes).await?; + self.state + .proof_cache .push(block_num, ProofNotification::new(block_num, proof_bytes)) .expect("proof cache receives sequential block numbers"); - self.proven_tip.advance(block_num); + self.state.proven_tip.advance(block_num); Ok(()) } } diff --git a/crates/store/src/state/writer/mod.rs b/crates/store/src/state/writer/mod.rs new file mode 100644 index 0000000000..063c23eafe --- /dev/null +++ b/crates/store/src/state/writer/mod.rs @@ -0,0 +1,138 @@ +//! Serialized block-write path for the store state. +//! +//! A single [`WriteWorker`] task owns the mutable trees and processes incoming [`WriteRequest`]s +//! one at a time via an mpsc channel. After each successful commit it publishes a new +//! [`StateSnapshot`] snapshot via an [`ArcSwap`], making the updated trees immediately visible to +//! wait-free readers. +//! +//! The [`BlockWriter`] and [`ProofWriter`] capabilities defined here are the only handles able +//! to feed this worker and to commit proofs; the submodules hold their entry points. + +mod apply_block; +mod apply_proof; + +mod worker; +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; + +use miden_node_utils::ErrorReport; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::block::SignedBlock; +use tokio::sync::{mpsc, oneshot}; +pub(in crate::state) use worker::WriteWorker; + +use crate::COMPONENT; +use crate::blocks::BlockStore; +use crate::errors::ApplyBlockError; + +// WRITE CAPABILITIES +// ================================================================================================ + +/// The store's block-write capability. +/// +/// Only handle able to apply blocks; obtained exactly once from [`LoadedState::start`](crate::state::LoadedState::start) and +/// deliberately not cloneable, so granting it to a single task (the block builder in sequencer +/// mode, the block sync loop in full-node mode) statically prevents every other component from +/// writing blocks. +/// +/// Exposes no read access: holders that also need to query the store receive the +/// [`Arc`](crate::state::State) returned alongside this capability by +/// [`LoadedState::start`](crate::state::LoadedState::start). +pub struct BlockWriter { + /// The block store, used to persist proving inputs alongside applied blocks. + pub(super) block_store: Arc, + /// Sender for block-write requests to the [`WriteWorker`] task. Never cloned out of this + /// struct: the writer exits once it is dropped. + pub(super) write_tx: mpsc::Sender, +} + +/// The store's proof-write capability. +/// +/// Only handle able to commit block proofs and advance the proven tip; obtained exactly once from +/// [`LoadedState::start`](crate::state::LoadedState::start) and deliberately not cloneable, so granting it to a single task (the +/// proof scheduler in sequencer mode, the proof sync loop in full-node mode) statically prevents +/// every other component from writing proofs. +/// +/// Exposes no read access: the held state is only used internally to commit proofs and advance +/// the proven tip. Holders that also need to query the store receive the +/// [`Arc`](crate::state::State) returned alongside this capability by +/// [`LoadedState::start`](crate::state::LoadedState::start). +pub struct ProofWriter { + pub(super) state: Arc, +} + +// WRITER TASK +// ================================================================================================ + +/// Handle of the store's write worker task, returned by [`LoadedState::start`](crate::state::LoadedState::start). +/// +/// Awaiting it resolves once the writer has exited and released the tree storage it owns; a join +/// error carries a writer panic. The newtype ensures [`BlockWriter::stop`] can only be given the +/// store's own writer task, and deliberately does not expose [`tokio::task::JoinHandle::abort`]: +/// aborting the writer mid-write could leave the trees lagging the committed database state, +/// voiding the guarantee that an in-flight block write always completes. +#[must_use = "await the writer task to observe its exit, or pass it to `BlockWriter::stop`"] +pub struct WriterTask(pub(super) tokio::task::JoinHandle<()>); + +impl Future for WriterTask { + type Output = Result<(), tokio::task::JoinError>; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + Pin::new(&mut self.0).poll(cx) + } +} + +// WRITE REQUEST +// ================================================================================================ + +/// A request to apply a block, paired with a one-shot channel for the result. +pub(super) struct WriteRequest { + signed_block: SignedBlock, + result_tx: oneshot::Sender>, +} + +impl BlockWriter { + /// Stops the store, waiting until the write worker has released the tree storage it owns. + /// + /// Consumes the capability — closing the write channel the write worker listens on — and then + /// joins the writer task returned by [`LoadedState::start`](crate::state::LoadedState::start). The drop must precede the join + /// or the write worker never observes the closed channel; doing both here keeps that ordering + /// out of caller hands. Read-only [`State`](crate::state::State) references may outlive the + /// stop. + /// + /// Callers that need the storage released deterministically must use this method instead of + /// dropping: the node's `recover` command stops the store before the process exits, and the + /// stress-test's store seeding stops it so the same data directory can be re-loaded (or its + /// temporary directory deleted) immediately afterwards. The running node does not use this + /// method — its writer exits via the shutdown token passed to + /// [`State::load`](crate::state::State::load) and is joined through the node's task set. + /// + /// # Panics + /// + /// Panics if the writer task panicked. + pub async fn stop(self, writer_task: WriterTask) { + drop(self); + writer_task.await.expect("write worker task should not panic"); + } + + /// Apply changes of a new block to the DB and in-memory data structures. + /// + /// Blocks are forwarded to the store's write worker task, which processes them one at a + /// time. + /// Readers are unaffected while a block is being applied: they keep reading from the previous + /// in-memory snapshot until the writer atomically publishes the new one. + #[miden_instrument( + target = COMPONENT, + err, + )] + pub async fn apply_block(&mut self, signed_block: SignedBlock) -> Result<(), ApplyBlockError> { + let (result_tx, result_rx) = oneshot::channel(); + self.write_tx + .send(WriteRequest { signed_block, result_tx }) + .await + .map_err(|e| ApplyBlockError::WriterTaskSendFailed(e.as_report()))?; + result_rx.await? + } +} diff --git a/crates/store/src/state/writer/worker.rs b/crates/store/src/state/writer/worker.rs new file mode 100644 index 0000000000..37879608fa --- /dev/null +++ b/crates/store/src/state/writer/worker.rs @@ -0,0 +1,523 @@ +//! The write worker: single-task owner of the store's mutable trees. + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use arc_swap::ArcSwap; +use miden_node_utils::ErrorReport; +use miden_node_utils::shutdown::CancellationToken; +use miden_node_utils::tracing::{miden_instrument, miden_span_record}; +use miden_protocol::Word; +use miden_protocol::account::AccountUpdateDetails; +use miden_protocol::block::account_tree::AccountMutationSet; +use miden_protocol::block::nullifier_tree::{NullifierMutationSet, NullifierTree}; +use miden_protocol::block::{BlockBody, BlockHeader, BlockNumber, Blockchain, SignedBlock}; +use miden_protocol::crypto::merkle::smt::LargeSmt; +use miden_protocol::note::{NoteDetails, Nullifier}; +use miden_protocol::transaction::OutputNote; +use miden_protocol::utils::serde::Serializable; +use tokio::sync::{mpsc, watch}; + +use super::WriteRequest; +use crate::account_state_forest::{ + AccountStateForest, + AccountStateForestBackend, + PreparedAccountStateForestBlockUpdate, +}; +use crate::accounts::AccountTreeWithHistory; +use crate::blocks::BlockStore; +use crate::db::{Db, NoteRecord}; +use crate::errors::{ApplyBlockError, InvalidBlockError}; +use crate::state::block_lifecycle::{BlockLifecycle, lifecycle_events_enabled}; +use crate::state::loader::TreeStorage; +use crate::state::view::{ + PublishedGenerations, + SNAPSHOTS_LIVE_WARN_THRESHOLD, + SnapshotGuard, + StateSnapshot, +}; +use crate::state::{BlockCache, BlockNotification}; +use crate::{COMPONENT, HistoricalError, LOG_TARGET}; + +// WRITE WORKER +// ================================================================================================ + +/// Single-task owner of the mutable trees. Processes [`WriteRequest`]s serially. +/// +/// The writer owns the writable trees directly, so no locks are held at any point: validation and +/// mutation-computation read the owned trees, the DB commit runs without touching them, and the +/// new [`StateSnapshot`] snapshot is published atomically at the end. +pub(in crate::state) struct WriteWorker { + db: Arc, + block_store: Arc, + /// Atomically swappable pointer through which new snapshots are published. + latest_snapshot: Arc>, + committed_tip_tx: Arc>, + block_cache: BlockCache, + rx: mpsc::Receiver, + /// The mutable nullifier tree owned by this writer. + nullifier_tree: NullifierTree>, + /// The mutable account tree owned by this writer. + account_tree: AccountTreeWithHistory, + /// The blockchain MMR owned by this writer. + blockchain: Blockchain, + /// The mutable account state forest owned by this writer. + forest: AccountStateForest, + /// Shared counter of live snapshot generations, for observability. + snapshots_live: Arc, + /// Writer-local log of published generations; its oldest still-pinned height feeds the + /// snapshot-aware history-pruning tip. + published_generations: PublishedGenerations, +} + +/// Note records and state mutations computed from a validated block, before any modifications. +struct PreparedBlockUpdate { + notes: Vec<(NoteRecord, Option)>, + nullifier_tree_update: NullifierMutationSet, + account_tree_update: AccountMutationSet, + account_forest_update: PreparedAccountStateForestBlockUpdate, +} + +impl WriteWorker { + /// Assembles the write worker from the loaded trees and the store's shared infrastructure. + /// + /// Only construction is exposed outside this module: once assembled, the worker's internals + /// are reachable solely through [`Self::run`]. + #[expect(clippy::too_many_arguments)] + pub(in crate::state) fn new( + db: Arc, + block_store: Arc, + latest_snapshot: Arc>, + committed_tip_tx: Arc>, + block_cache: BlockCache, + rx: mpsc::Receiver, + nullifier_tree: NullifierTree>, + account_tree: AccountTreeWithHistory, + blockchain: Blockchain, + forest: AccountStateForest, + snapshots_live: Arc, + ) -> Self { + // Seed the generation log with the initial snapshot so its readers hold back pruning + // exactly like readers of any later generation. + let mut published_generations = PublishedGenerations::new(); + let initial_snapshot = latest_snapshot.load_full(); + published_generations.record(initial_snapshot.latest_block_num(), &initial_snapshot); + Self { + db, + block_store, + latest_snapshot, + committed_tip_tx, + block_cache, + rx, + nullifier_tree, + account_tree, + blockchain, + forest, + snapshots_live, + published_generations, + } + } + + /// Runs the writer loop, processing requests until `shutdown` is signalled or the + /// [`BlockWriter`] (holding the only request sender) is dropped. + /// + /// Cancellation is only observed between requests: an in-flight block write always runs to + /// completion, so shutdown never leaves the trees lagging the committed database state. + /// Requests still queued when cancellation fires are dropped, failing their senders. + pub async fn run(mut self, shutdown: CancellationToken) { + loop { + let req = tokio::select! { + biased; + () = shutdown.cancelled() => break, + req = self.rx.recv() => match req { + Some(req) => req, + None => break, + }, + }; + let result = self.write_block(req.signed_block).await; + let _ = req.result_tx.send(result); + } + } + + /// Validates and commits a signed block to all persistent and in-memory stores. + /// + /// ## Note on state consistency + /// + /// Readers access the in-memory state through frozen snapshots, so consistency is maintained + /// by ordering the commit steps rather than by locking: + /// + /// - the block is validated against the writer-owned trees and the DB prior to starting any + /// modifications. + /// - the block is saved to the block store. Such blocks are considered candidates and are not + /// yet available for reading because the latest block pointer is not updated yet. + /// - the DB transaction is committed. Concurrent readers still see the previous in-memory + /// snapshot; queries that combine DB and in-memory data are scoped by block number. + /// - the in-memory structures owned by the writer are updated. On a crash in between, the trees + /// lag the DB by one block, which is detected by the consistency checks at startup (the same + /// crash semantics as the previous lock-based implementation). + /// - the new snapshot is published atomically, making the block visible to readers. + #[miden_instrument( + target = COMPONENT, + err, + )] + async fn write_block(&mut self, signed_block: SignedBlock) -> Result<(), ApplyBlockError> { + let header = signed_block.header(); + let body = signed_block.body(); + + let block_num = header.block_num(); + let block_commitment = header.commitment(); + let num_transactions = body.transactions().as_slice().len(); + + miden_span_record!( + block.number = %block_num, + block.commitment = %block_commitment, + block.transactions.count = num_transactions, + ); + + self.validate_block_header(header, body).await?; + + let block_lifecycle = + lifecycle_events_enabled().then(|| BlockLifecycle::from_block_body(block_num, body)); + let unresolved_note_nullifiers = block_lifecycle + .as_ref() + .map_or_else(Vec::new, BlockLifecycle::unresolved_note_nullifiers); + + // Compute the tree and forest mutations and note records upfront, before any modifications. + // The writer is the sole forest mutator, so the precomputed forest update stays valid until + // it is applied after the DB commit below. + let PreparedBlockUpdate { + notes, + nullifier_tree_update, + account_tree_update, + account_forest_update, + } = tokio::task::block_in_place(|| self.prepare_block_update(header, body))?; + let precomputed_public_states = account_forest_update.account_states.clone(); + + // Save the block to the block store. In a case of a failed DB transaction, the in-memory + // state will be unchanged, but the file might still be written. Such blocks should be + // considered candidates, not finalized blocks. + let signed_block_bytes = signed_block.to_bytes(); + self.block_store.save_block(block_num, &signed_block_bytes).await?; + + // Commit to the DB. Readers continue to see the previous in-memory snapshot while the DB + // commits; queries that combine DB and in-memory data are scoped by block number. + // + // History pruning runs inside the same DB transaction, keyed off the oldest live snapshot + // generation rather than the actual tip: unlike the `RocksDB`-backed trees, SQLite reads + // have no point-in-time protection, so pruning lags while pinned views can still reach + // the history and catches up once they are released. + let prune_tip = self.published_generations.prune_tip(block_num); + let resolved_note_ids = self + .db + .apply_block( + signed_block, + notes, + precomputed_public_states, + unresolved_note_nullifiers, + prune_tip, + ) + .await + .map_err(|err| ApplyBlockError::DbUpdateTaskFailed(err.as_report()))?; + + // The DB is committed at this point, so the prepared mutations must be applied and any + // failure to do so aborts the process. + let snapshot = tokio::task::block_in_place(|| { + self.apply_prepared_mutations( + block_num, + block_commitment, + nullifier_tree_update, + account_tree_update, + account_forest_update, + ) + }); + + // Atomically publish the new state. Readers that call `snapshot()` after this point will + // see the updated state. Readers holding the old snapshot continue unaffected, but are on + // the clock: a superseded generation held too long is reported on release. + self.published_generations.record(block_num, &snapshot); + self.latest_snapshot.swap(snapshot).mark_superseded(); + + let snapshots_live = self.check_live_snapshots(block_num); + miden_span_record!(snapshots.live = snapshots_live); + + // Push to cache and notify replica subscribers. + self.block_cache + .push(block_num, BlockNotification::new(block_num, signed_block_bytes)) + .expect("block cache receives sequential block numbers"); + // `send` is a no-op (and reports an error) when there are no subscribers, which would leave + // `committed_tip()` stuck reporting a stale value. Use `send_replace` so the tip is always + // updated regardless of whether anything is currently subscribed. + self.committed_tip_tx.send_replace(block_num); + + if let Some(block_lifecycle) = block_lifecycle { + block_lifecycle.emit(&resolved_note_ids); + } + tracing::debug!(target: LOG_TARGET, "Block applied"); + + Ok(()) + } + + /// Returns the number of live snapshot generations, warning when slow readers are pinning too + /// many old generations in memory. + /// + /// The count is returned rather than recorded here because `miden_span_record!` must be used + /// within a `#[miden_instrument]` function. + fn check_live_snapshots(&self, block_num: BlockNumber) -> u64 { + let snapshots_live = self.snapshots_live.load(Ordering::Relaxed) as u64; + if snapshots_live > SNAPSHOTS_LIVE_WARN_THRESHOLD { + tracing::warn!( + target: COMPONENT, + block_num = block_num.as_u32(), + snapshots.live = snapshots_live, + "too many live state snapshots; slow readers are pinning old generations", + ); + } + snapshots_live + } + + /// Computes the note records and all tree and forest mutations for a block, without mutating + /// any state. + /// + /// May block on backend I/O, so it must run on Tokio's blocking path. The returned forest + /// update is bound to the forest state observed here; it remains valid until applied because + /// the writer is the sole forest mutator. + fn prepare_block_update( + &self, + header: &BlockHeader, + body: &BlockBody, + ) -> Result { + let notes = Self::build_note_records(header, body)?; + let (nullifier_tree_update, account_tree_update) = + self.compute_tree_mutations(header, body)?; + + // Public account updates carry patches; private accounts are filtered out since they don't + // expose their state changes. + let account_patches = + body.updated_accounts().iter().filter_map(|update| match update.details() { + AccountUpdateDetails::Public(patch) => Some(patch.clone()), + AccountUpdateDetails::Private => None, + }); + let account_forest_update = self + .forest + .compute_block_update_mutations(header.block_num(), account_patches) + .map_err(ApplyBlockError::AccountStateForestPreparation)?; + + Ok(PreparedBlockUpdate { + notes, + nullifier_tree_update, + account_tree_update, + account_forest_update, + }) + } + + /// Applies the prepared mutations to the writer-owned trees and builds the new snapshot from + /// reader views of them. The reader views are point-in-time storage snapshots, so no tree data + /// is copied. + /// + /// Must only be called after the corresponding DB commit: at that point the mutations are part + /// of canonical state, so a failure to apply them leaves the trees divergent and panics. + /// Returning an error instead would expose components at different block heights. The panic + /// unwinds the writer task, whose join error shuts the node down; readers keep serving the + /// previous published snapshot (block-scoped, so still consistent) until then, and the startup + /// consistency checks detect the trees lagging the database on restart. + /// + /// May block on backend I/O, so it must run on Tokio's blocking path. + /// + /// # Panics + /// + /// Panics if applying any prepared mutation fails; see above. + fn apply_prepared_mutations( + &mut self, + block_num: BlockNumber, + block_commitment: Word, + nullifier_tree_update: NullifierMutationSet, + account_tree_update: AccountMutationSet, + account_forest_update: PreparedAccountStateForestBlockUpdate, + ) -> Arc { + self.nullifier_tree + .apply_mutations(nullifier_tree_update) + .unwrap_or_else(|error| { + panic!("nullifier tree update failed after database commit: {error}") + }); + + self.account_tree.apply_mutations(account_tree_update).unwrap_or_else(|error| { + panic!("account tree update failed after database commit: {error}") + }); + + self.blockchain.push(block_commitment); + + self.forest + .apply_precomputed_block_update(block_num, account_forest_update) + .unwrap_or_else(|error| { + panic!("account-state forest update failed after database commit: {error}") + }); + + Arc::new(StateSnapshot::new( + self.nullifier_tree + .reader() + .expect("nullifier tree snapshot creation should not fail"), + self.blockchain.clone(), + self.account_tree.reader(), + self.forest.reader().expect("forest snapshot creation should not fail"), + SnapshotGuard::new(Arc::clone(&self.snapshots_live), block_num), + )) + } + + /// Validates that the block header is consistent with the block body and the current state. + #[miden_instrument( + target = COMPONENT, + err, + )] + async fn validate_block_header( + &self, + header: &BlockHeader, + body: &BlockBody, + ) -> Result<(), ApplyBlockError> { + // Validate that header and body match. + let tx_commitment = body.transactions().commitment(); + if header.tx_commitment() != tx_commitment { + return Err(InvalidBlockError::InvalidBlockTxCommitment { + expected: tx_commitment, + actual: header.tx_commitment(), + } + .into()); + } + + let block_num = header.block_num(); + + // Validate that the applied block is the next block in sequence. + let prev_block = self + .db + .select_block_header_by_block_num(None) + .await? + .ok_or(ApplyBlockError::DbBlockHeaderEmpty)?; + let expected_block_num = prev_block.block_num().child(); + if block_num != expected_block_num { + return Err(InvalidBlockError::NewBlockInvalidBlockNum { + expected: expected_block_num, + submitted: block_num, + } + .into()); + } + if header.prev_block_commitment() != prev_block.commitment() { + return Err(InvalidBlockError::NewBlockInvalidPrevCommitment.into()); + } + + Ok(()) + } + + /// Computes nullifier and account tree mutations, validating roots against the block header. + #[miden_instrument( + target = COMPONENT, + err, + )] + fn compute_tree_mutations( + &self, + header: &BlockHeader, + body: &BlockBody, + ) -> Result<(NullifierMutationSet, AccountMutationSet), ApplyBlockError> { + let block_num = header.block_num(); + + // A nullifier can only ever be created once, so the block is invalid if any of its + // nullifiers are already recorded in the tree. + let duplicate_nullifiers: Vec<_> = body + .created_nullifiers() + .iter() + .filter(|&nullifier| self.nullifier_tree.get_block_num(nullifier).is_some()) + .copied() + .collect(); + if !duplicate_nullifiers.is_empty() { + return Err(InvalidBlockError::DuplicatedNullifiers(duplicate_nullifiers).into()); + } + + // The header's chain commitment must equal the chain MMR root prior to this block. + let peaks = self.blockchain.peaks(); + if peaks.hash_peaks() != header.chain_commitment() { + return Err(InvalidBlockError::NewBlockInvalidChainCommitment.into()); + } + + // Compute the nullifier tree mutations and verify that they produce the nullifier root + // claimed in the header. + let nullifier_tree_update = self + .nullifier_tree + .compute_mutations( + body.created_nullifiers().iter().map(|nullifier| (*nullifier, block_num)), + ) + .map_err(InvalidBlockError::NewBlockNullifierAlreadySpent)?; + + if nullifier_tree_update.as_mutation_set().root() != header.nullifier_root() { + return Err(InvalidBlockError::NewBlockInvalidNullifierRoot.into()); + } + + // Compute the account tree mutations and verify that they produce the account root claimed + // in the header. + let account_tree_update = self + .account_tree + .compute_mutations( + body.updated_accounts() + .iter() + .map(|update| (update.account_id(), update.final_state_commitment())), + ) + .map_err(|e| match e { + HistoricalError::AccountTreeError(err) => { + InvalidBlockError::NewBlockDuplicateAccountIdPrefix(err) + }, + HistoricalError::MerkleError(_) => { + panic!("Unexpected MerkleError during account tree mutation computation") + }, + })?; + + if account_tree_update.as_mutation_set().root() != header.account_root() { + return Err(InvalidBlockError::NewBlockInvalidAccountRoot.into()); + } + + Ok((nullifier_tree_update, account_tree_update)) + } + + /// Builds note records with inclusion proofs from the block body. + #[miden_instrument( + target = COMPONENT, + err, + )] + fn build_note_records( + header: &BlockHeader, + body: &BlockBody, + ) -> Result)>, ApplyBlockError> { + let block_num = header.block_num(); + + let note_tree = body.compute_block_note_tree(); + if note_tree.root() != header.note_root() { + return Err(InvalidBlockError::NewBlockInvalidNoteRoot.into()); + } + + let notes = body + .output_notes() + .map(|(note_index, note)| { + let (details, attachments, nullifier) = match note { + OutputNote::Public(public) => ( + Some(NoteDetails::from(public.as_note())), + public.as_note().attachments().clone(), + Some(public.as_note().nullifier()), + ), + OutputNote::Private(private) => (None, private.attachments().clone(), None), + }; + + let inclusion_path = note_tree.open(note_index); + + let note_record = NoteRecord { + block_num, + note_index, + note_id: note.id().as_word(), + metadata: *note.metadata(), + details, + attachments, + inclusion_path, + }; + + Ok((note_record, nullifier)) + }) + .collect::, InvalidBlockError>>()?; + + Ok(notes) + } +} diff --git a/crates/tracing-macro/src/lib.rs b/crates/tracing-macro/src/lib.rs index 07ce7fa80d..ee76b6ff61 100644 --- a/crates/tracing-macro/src/lib.rs +++ b/crates/tracing-macro/src/lib.rs @@ -81,6 +81,9 @@ const ALLOWED_FIELD_NAMES: &[&str] = &[ "reference_block.number", "request.kind", "script.root", + "snapshot.block_num", + "snapshot.lifetime_ms", + "snapshots.live", "transaction.id", "transaction.expires_at", "transaction.input_notes.count", diff --git a/crates/utils/src/retry.rs b/crates/utils/src/retry.rs index 245227539a..19db8e0000 100644 --- a/crates/utils/src/retry.rs +++ b/crates/utils/src/retry.rs @@ -16,7 +16,7 @@ use std::time::Duration; -pub use backon::{BackoffBuilder, Retryable}; +pub use backon::{BackoffBuilder, Retryable, RetryableWithContext}; use backon::{ConstantBuilder, ExponentialBuilder}; // BACKOFF BUILDERS diff --git a/crates/utils/src/tasks.rs b/crates/utils/src/tasks.rs index b5c7e88e7d..eb4c4ade9b 100644 --- a/crates/utils/src/tasks.rs +++ b/crates/utils/src/tasks.rs @@ -89,9 +89,17 @@ impl Tasks { /// Waits for either an unexpected task completion or a shutdown request. /// /// Before shutdown, any task completion is treated as fatal because this type supervises - /// long-running tasks. Once `token` is cancelled, clean task exits are accepted and this method - /// waits for all tracked tasks to finish. + /// long-running tasks. Such a completion triggers the shutdown itself: the token is cancelled + /// and the remaining tasks are drained before the error is returned. Returning without + /// draining would drop the set and abort the surviving tasks mid-work — e.g. the store's + /// block writer between its database commit and tree update, tearing persistent state. + /// + /// Once `token` is cancelled (whether externally or by a failure here), clean task exits are + /// accepted and this method waits for all tracked tasks to finish. The first failure observed + /// is returned as the root cause; subsequent failures are logged, since they are often + /// knock-on effects of the first. pub async fn join_next_or_cancelled(&mut self, token: CancellationToken) -> anyhow::Result<()> { + let mut outcome = Ok(()); while !token.is_cancelled() { tokio::select! { biased; @@ -100,18 +108,35 @@ impl Tasks { let Some((task, result)) = result else { anyhow::bail!("task set is empty"); }; - Self::unexpected_completion(&task, result)?; + outcome = Self::unexpected_completion(&task, result); + // Shut the remaining tasks down and fall through to the drain below. + token.cancel(); }, } } while let Some((task, result)) = self.join_next().await { - Self::shutdown_completion(&task, result)?; + match (&outcome, Self::shutdown_completion(&task, result)) { + // No failure so far: this task's result (clean or failed) becomes the outcome. + (Ok(()), result) => outcome = result, + // A failure is already recorded as the root cause; later failures are often + // knock-on effects of it, so log them rather than mask it. + (Err(_), Err(err)) => { + tracing::warn!(task = %task, error = %format!("{err:#}"), "task failed during shutdown"); + }, + // A failure is already recorded and this task exited cleanly: nothing to add. + (Err(_), Ok(())) => {}, + } } - Ok(()) + outcome } + /// Interprets a task completion observed *before* shutdown was requested. + /// + /// Supervised tasks are expected to run until shutdown, so every completion — even a clean + /// exit — is an error here; the variants only differ in how much context the error carries + /// (task failure, or a panicked/aborted task surfacing as a [`JoinError`]). fn unexpected_completion( task: &str, result: Result, JoinError>, @@ -123,6 +148,11 @@ impl Tasks { } } + /// Interprets a task completion observed *after* shutdown was requested. + /// + /// During shutdown a clean exit is the expected outcome, and a cancelled task is also fine — + /// abort is how a dropped set winds tasks down. A task error or a panic (a non-cancellation + /// [`JoinError`]) is still a failure worth reporting. fn shutdown_completion( task: &str, result: Result, JoinError>, @@ -176,6 +206,84 @@ mod tests { assert_eq!(err.to_string(), "task worker completed unexpectedly"); } + #[tokio::test] + async fn join_next_or_cancelled_drains_remaining_tasks_after_a_failure() { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + let token = crate::shutdown::CancellationToken::new(); + let mut tasks = Tasks::new(); + let survivor_finished = Arc::new(AtomicBool::new(false)); + + tasks.spawn("failing", async { anyhow::bail!("boom") }); + tasks.spawn("survivor", { + let token = token.clone(); + let finished = Arc::clone(&survivor_finished); + async move { + token.cancelled().await; + // Work past the cancellation point: an aborted task would never get here. + tokio::time::sleep(Duration::from_millis(10)).await; + finished.store(true, Ordering::Relaxed); + Ok(()) + } + }); + + let err = tasks + .join_next_or_cancelled(token.clone()) + .await + .expect_err("the failing task's error should be returned"); + + assert_eq!(err.to_string(), "task failing failed"); + assert!(token.is_cancelled(), "a task failure should trigger shutdown"); + assert!(tasks.is_empty(), "all tasks should be drained before returning"); + assert!( + survivor_finished.load(Ordering::Relaxed), + "surviving tasks should shut down gracefully, not be aborted", + ); + } + + #[tokio::test] + async fn join_next_or_cancelled_drains_past_failures_during_shutdown() { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + let token = crate::shutdown::CancellationToken::new(); + let mut tasks = Tasks::new(); + let survivor_finished = Arc::new(AtomicBool::new(false)); + + tasks.spawn("failing", { + let token = token.clone(); + async move { + token.cancelled().await; + anyhow::bail!("boom") + } + }); + tasks.spawn("survivor", { + let token = token.clone(); + let finished = Arc::clone(&survivor_finished); + async move { + token.cancelled().await; + tokio::time::sleep(Duration::from_millis(10)).await; + finished.store(true, Ordering::Relaxed); + Ok(()) + } + }); + + token.cancel(); + + let err = tasks + .join_next_or_cancelled(token) + .await + .expect_err("a failure during shutdown should be reported"); + + assert_eq!(err.to_string(), "task failing failed during shutdown"); + assert!(tasks.is_empty(), "draining should continue past the failed task"); + assert!( + survivor_finished.load(Ordering::Relaxed), + "surviving tasks should shut down gracefully, not be aborted", + ); + } + #[tokio::test] async fn join_next_or_cancelled_waits_for_all_tasks_to_complete_after_cancellation() { let token = crate::shutdown::CancellationToken::new(); diff --git a/crates/utils/tests/ui/tracing_macros/invalid_field_name.stderr b/crates/utils/tests/ui/tracing_macros/invalid_field_name.stderr index 14c89f9db3..403f90c3eb 100644 --- a/crates/utils/tests/ui/tracing_macros/invalid_field_name.stderr +++ b/crates/utils/tests/ui/tracing_macros/invalid_field_name.stderr @@ -1,4 +1,4 @@ -error: unsupported tracing field `tx_id`; use one of: account.id, account.id.network_prefix, account.ids, account.ids.count, account.updated, batch.id, batch.account_updates.count, batch.expires_at, batch.expiration_height, batch.input_notes.count, batch.output_notes.count, batch.reference_block.commitment, batch.reference_block.number, block.batch.ids, block.batches.count, block.batches.output_notes.count, block.commitment, block.commitments.account, block.commitments.chain, block.commitments.kernel, block.commitments.note, block.commitments.nullifier, block.commitments.transaction, block.erased_note_proofs.count, block.erased_notes.count, block.from, block.nullifiers.count, block.number, block.output_notes.count, block.prev_block_commitment, block.protocol.version, block.size, block.sub_commitment, block.timestamp, block.transactions.ids, block.transactions.count, block.updated_accounts.count, block_range.from, block_range.to, current_client_block_height, cutoff_block, db.account_state_forest.size, db.account_tree.size, db.block_store.size, db.nullifier_tree.size, db.sqlite.size, db.sqlite.wal.size, dice_roll, failure_rate, finality_level, inputs_size, mempool.accounts, mempool.batches.proposed, mempool.batches.proven, mempool.nullifiers, mempool.output_notes, mempool.transactions.unbatched, mempool.transactions.uncommitted, note.id, notes.count, nullifiers, path, port, prefix_len, prefixes, proof_size, prover, prover.kind, reference_block.number, request.kind, script.root, transaction.id, transaction.expires_at, transaction.input_notes.count, transaction.output_notes.count, transaction.reference_block.commitment, transaction.reference_block.number, tip.number, transactions.count, transactions.ids, transactions.input_notes.count, transactions.output_notes.count, transactions.unauthenticated_notes.count, workers.active, workers.capacity, workers.count +error: unsupported tracing field `tx_id`; use one of: account.id, account.id.network_prefix, account.ids, account.ids.count, account.updated, batch.id, batch.account_updates.count, batch.expires_at, batch.expiration_height, batch.input_notes.count, batch.output_notes.count, batch.reference_block.commitment, batch.reference_block.number, block.batch.ids, block.batches.count, block.batches.output_notes.count, block.commitment, block.commitments.account, block.commitments.chain, block.commitments.kernel, block.commitments.note, block.commitments.nullifier, block.commitments.transaction, block.erased_note_proofs.count, block.erased_notes.count, block.from, block.nullifiers.count, block.number, block.output_notes.count, block.prev_block_commitment, block.protocol.version, block.size, block.sub_commitment, block.timestamp, block.transactions.ids, block.transactions.count, block.updated_accounts.count, block_range.from, block_range.to, current_client_block_height, cutoff_block, db.account_state_forest.size, db.account_tree.size, db.block_store.size, db.nullifier_tree.size, db.sqlite.size, db.sqlite.wal.size, dice_roll, failure_rate, finality_level, inputs_size, mempool.accounts, mempool.batches.proposed, mempool.batches.proven, mempool.nullifiers, mempool.output_notes, mempool.transactions.unbatched, mempool.transactions.uncommitted, note.id, notes.count, nullifiers, path, port, prefix_len, prefixes, proof_size, prover, prover.kind, reference_block.number, request.kind, script.root, snapshot.block_num, snapshot.lifetime_ms, snapshots.live, transaction.id, transaction.expires_at, transaction.input_notes.count, transaction.output_notes.count, transaction.reference_block.commitment, transaction.reference_block.number, tip.number, transactions.count, transactions.ids, transactions.input_notes.count, transactions.output_notes.count, transactions.unauthenticated_notes.count, workers.active, workers.capacity, workers.count --> tests/ui/tracing_macros/invalid_field_name.rs:8:9 | 8 | tx_id = %tx_id, diff --git a/crates/utils/tests/ui/tracing_macros/invalid_instrument_field_name.stderr b/crates/utils/tests/ui/tracing_macros/invalid_instrument_field_name.stderr index 1f4bafe59a..e37f59b726 100644 --- a/crates/utils/tests/ui/tracing_macros/invalid_instrument_field_name.stderr +++ b/crates/utils/tests/ui/tracing_macros/invalid_instrument_field_name.stderr @@ -1,4 +1,4 @@ -error: unsupported tracing field `tx_id`; use one of: account.id, account.id.network_prefix, account.ids, account.ids.count, account.updated, batch.id, batch.account_updates.count, batch.expires_at, batch.expiration_height, batch.input_notes.count, batch.output_notes.count, batch.reference_block.commitment, batch.reference_block.number, block.batch.ids, block.batches.count, block.batches.output_notes.count, block.commitment, block.commitments.account, block.commitments.chain, block.commitments.kernel, block.commitments.note, block.commitments.nullifier, block.commitments.transaction, block.erased_note_proofs.count, block.erased_notes.count, block.from, block.nullifiers.count, block.number, block.output_notes.count, block.prev_block_commitment, block.protocol.version, block.size, block.sub_commitment, block.timestamp, block.transactions.ids, block.transactions.count, block.updated_accounts.count, block_range.from, block_range.to, current_client_block_height, cutoff_block, db.account_state_forest.size, db.account_tree.size, db.block_store.size, db.nullifier_tree.size, db.sqlite.size, db.sqlite.wal.size, dice_roll, failure_rate, finality_level, inputs_size, mempool.accounts, mempool.batches.proposed, mempool.batches.proven, mempool.nullifiers, mempool.output_notes, mempool.transactions.unbatched, mempool.transactions.uncommitted, note.id, notes.count, nullifiers, path, port, prefix_len, prefixes, proof_size, prover, prover.kind, reference_block.number, request.kind, script.root, transaction.id, transaction.expires_at, transaction.input_notes.count, transaction.output_notes.count, transaction.reference_block.commitment, transaction.reference_block.number, tip.number, transactions.count, transactions.ids, transactions.input_notes.count, transactions.output_notes.count, transactions.unauthenticated_notes.count, workers.active, workers.capacity, workers.count +error: unsupported tracing field `tx_id`; use one of: account.id, account.id.network_prefix, account.ids, account.ids.count, account.updated, batch.id, batch.account_updates.count, batch.expires_at, batch.expiration_height, batch.input_notes.count, batch.output_notes.count, batch.reference_block.commitment, batch.reference_block.number, block.batch.ids, block.batches.count, block.batches.output_notes.count, block.commitment, block.commitments.account, block.commitments.chain, block.commitments.kernel, block.commitments.note, block.commitments.nullifier, block.commitments.transaction, block.erased_note_proofs.count, block.erased_notes.count, block.from, block.nullifiers.count, block.number, block.output_notes.count, block.prev_block_commitment, block.protocol.version, block.size, block.sub_commitment, block.timestamp, block.transactions.ids, block.transactions.count, block.updated_accounts.count, block_range.from, block_range.to, current_client_block_height, cutoff_block, db.account_state_forest.size, db.account_tree.size, db.block_store.size, db.nullifier_tree.size, db.sqlite.size, db.sqlite.wal.size, dice_roll, failure_rate, finality_level, inputs_size, mempool.accounts, mempool.batches.proposed, mempool.batches.proven, mempool.nullifiers, mempool.output_notes, mempool.transactions.unbatched, mempool.transactions.uncommitted, note.id, notes.count, nullifiers, path, port, prefix_len, prefixes, proof_size, prover, prover.kind, reference_block.number, request.kind, script.root, snapshot.block_num, snapshot.lifetime_ms, snapshots.live, transaction.id, transaction.expires_at, transaction.input_notes.count, transaction.output_notes.count, transaction.reference_block.commitment, transaction.reference_block.number, tip.number, transactions.count, transactions.ids, transactions.input_notes.count, transactions.output_notes.count, transactions.unauthenticated_notes.count, workers.active, workers.capacity, workers.count --> tests/ui/tracing_macros/invalid_instrument_field_name.rs:5:9 | 5 | tx_id = %"0x1234",