From 3b6bde5761470a467c6eef128c5f9fcae678d152 Mon Sep 17 00:00:00 2001 From: Stan Bondi Date: Tue, 19 Aug 2025 13:52:08 +0400 Subject: [PATCH 1/5] refactor(state_sync)!: sync state transitions by (shard,state_version) --- Cargo.lock | 12 + Cargo.toml | 1 + applications/tari_indexer/src/block_data.rs | 4 +- .../tari_indexer/src/event_scanner.rs | 6 +- applications/tari_validator_node/Cargo.toml | 1 + .../tari_validator_node/src/bootstrap.rs | 15 +- .../src/p2p/rpc/block_sync_task.rs | 4 +- .../src/p2p/rpc/service_impl.rs | 62 ++- .../src/p2p/rpc/state_sync_task.rs | 120 ++--- .../src/state_bootstrap.rs | 124 +---- crates/common_types/Cargo.toml | 1 + crates/common_types/src/committee.rs | 4 +- crates/common_types/src/lib.rs | 2 + crates/common_types/src/num_preshards.rs | 1 + crates/common_types/src/shard.rs | 15 +- crates/common_types/src/shard_group.rs | 8 +- .../common_types/src/shard_state_versions.rs | 180 +++++++ crates/common_types/src/substate_address.rs | 84 +-- .../common_types/src/versioned_substate_id.rs | 10 + .../src/hotstuff/block_change_set.rs | 24 +- .../consensus/src/hotstuff/commit_proofs.rs | 5 +- crates/consensus/src/hotstuff/common.rs | 25 +- crates/consensus/src/hotstuff/error.rs | 11 +- crates/consensus/src/hotstuff/on_propose.rs | 8 +- .../on_ready_to_vote_on_local_block.rs | 36 +- .../src/hotstuff/on_receive_local_proposal.rs | 14 +- .../src/hotstuff/state_machine/running.rs | 1 + .../hotstuff/substate_store/pending_store.rs | 71 ++- .../substate_store/shard_state_store.rs | 27 +- .../substate_store/sharded_state_tree.rs | 20 +- crates/consensus/src/hotstuff/worker.rs | 20 +- crates/consensus_tests/fixtures/block.json | 2 +- .../fixtures/block_with_dummies.json | 2 +- crates/consensus_tests/src/consensus.rs | 46 +- crates/consensus_tests/src/dummy_blocks.rs | 11 +- crates/consensus_tests/src/state_tree.rs | 55 +- crates/consensus_tests/src/substate_store.rs | 32 +- crates/consensus_tests/src/support/harness.rs | 42 +- crates/consensus_tests/src/support/helpers.rs | 1 + crates/p2p/proto/consensus.proto | 30 +- crates/p2p/proto/rpc.proto | 37 +- crates/p2p/src/conversions/consensus.rs | 89 ++- crates/p2p/src/conversions/rpc.rs | 119 ++-- crates/rpc_state_sync/src/state_sync.rs | 507 +++++++++--------- crates/state_store_rocksdb/src/cf_api.rs | 45 +- crates/state_store_rocksdb/src/codecs/mod.rs | 2 +- .../state_store_rocksdb/src/codecs/tuple.rs | 32 +- .../src/column_families/block_diff.rs | 4 +- .../src/column_families/bookkeeping.rs | 22 +- .../src/column_families/state_transition.rs | 80 +-- .../src/column_families/state_tree.rs | 15 +- .../state_tree_shard_versions.rs | 10 +- .../src/dbs/transaction.rs | 6 +- crates/state_store_rocksdb/src/lib.rs | 1 + crates/state_store_rocksdb/src/range.rs | 35 ++ crates/state_store_rocksdb/src/reader.rs | 174 +++--- crates/state_store_rocksdb/src/store.rs | 2 - crates/state_store_rocksdb/src/writer.rs | 185 +++---- crates/state_store_tests/src/block_diffs.rs | 6 +- crates/state_store_tests/src/blocks.rs | 12 +- .../src/foreign_proposals.rs | 6 +- crates/state_store_tests/src/helpers.rs | 77 ++- crates/state_store_tests/src/misc.rs | 11 +- .../src/missing_transactions.rs | 4 +- .../src/state_transitions.rs | 135 ++--- crates/state_store_tests/src/state_tree.rs | 4 +- crates/state_store_tests/src/substates.rs | 55 +- crates/state_store_tests/src/transactions.rs | 4 +- crates/state_tree/src/lib.rs | 3 + crates/state_tree/src/tree.rs | 29 +- crates/state_tree/tests/support.rs | 12 +- crates/state_tree/tests/test.rs | 7 +- crates/storage/src/consensus_models/block.rs | 167 ++++-- .../src/consensus_models/block_diff.rs | 8 +- .../src/consensus_models/block_header.rs | 44 +- .../src/consensus_models/epoch_checkpoint.rs | 47 +- .../src/consensus_models/epoch_state_root.rs | 60 --- crates/storage/src/consensus_models/mod.rs | 4 +- .../src/consensus_models/state_transition.rs | 180 ++----- .../src/consensus_models/state_tree_diff.rs | 10 +- .../storage/src/consensus_models/substate.rs | 166 +++--- .../src/consensus_models/substate_change.rs | 14 + .../consensus_models/substate_update_batch.rs | 55 ++ crates/storage/src/state_store/mod.rs | 53 +- .../src/webserver/handlers/bookkeeping.rs | 8 +- .../webserver/handlers/state_transitions.rs | 30 +- .../db_inspector/src/webserver/server.rs | 1 - 87 files changed, 2004 insertions(+), 1710 deletions(-) create mode 100644 crates/common_types/src/shard_state_versions.rs create mode 100644 crates/state_store_rocksdb/src/range.rs delete mode 100644 crates/storage/src/consensus_models/epoch_state_root.rs create mode 100644 crates/storage/src/consensus_models/substate_update_batch.rs diff --git a/Cargo.lock b/Cargo.lock index f49256e889..4df70eb20d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1239,6 +1239,16 @@ dependencies = [ "syn 2.0.103", ] +[[package]] +name = "bounded-vec" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09dc0086e469182132244e9b8d313a0742e1132da43a08c24b9dd3c18e0faf3a" +dependencies = [ + "serde", + "thiserror 2.0.12", +] + [[package]] name = "bs58" version = "0.4.0" @@ -11453,6 +11463,7 @@ version = "0.11.2" dependencies = [ "blake2", "borsh", + "bounded-vec", "digest", "ethnum", "indexmap 2.9.0", @@ -12217,6 +12228,7 @@ dependencies = [ "tari_shutdown", "tari_sidechain", "tari_state_store_rocksdb", + "tari_state_tree", "tari_swarm", "tari_template_builtin", "tari_template_lib", diff --git a/Cargo.toml b/Cargo.toml index 6a7fb0516d..b268360ad2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -165,6 +165,7 @@ blake2 = "0.10.6" borsh = { version = "1.5", default-features = false } bytes = "1.10.0" bnum = "0.13.0" +bounded-vec = "0.9.0" cacache = "12.0.0" cargo_toml = "0.20.5" ciborium = { version = "0.2.2", default-features = false } diff --git a/applications/tari_indexer/src/block_data.rs b/applications/tari_indexer/src/block_data.rs index 388e608ef4..dfce516e2d 100644 --- a/applications/tari_indexer/src/block_data.rs +++ b/applications/tari_indexer/src/block_data.rs @@ -1,10 +1,10 @@ // Copyright 2025 The Tari Project // SPDX-License-Identifier: BSD-3-Clause -use tari_ootle_storage::consensus_models::{Block, SubstateUpdate}; +use tari_ootle_storage::consensus_models::{Block, SubstateUpdateProof}; #[derive(Clone, Debug)] pub struct BlockData { pub block: Block, - pub diff: Vec, + pub diff: Vec, } diff --git a/applications/tari_indexer/src/event_scanner.rs b/applications/tari_indexer/src/event_scanner.rs index 8d93bc7db9..85e0fcddbe 100644 --- a/applications/tari_indexer/src/event_scanner.rs +++ b/applications/tari_indexer/src/event_scanner.rs @@ -33,7 +33,7 @@ use tari_engine_types::{ use tari_epoch_manager::{service::EpochManagerHandle, EpochManagerReader}; use tari_ootle_common_types::{committee::Committee, Epoch, PeerAddress, ShardGroup}; use tari_ootle_p2p::{proto, proto::rpc::SyncBlocksRequest}; -use tari_ootle_storage::consensus_models::{Block, SubstateUpdate}; +use tari_ootle_storage::consensus_models::{Block, SubstateUpdateProof}; use tari_template_lib::types::{EntityId, TemplateAddress}; use tari_template_manager::interface::{TemplateChange, TemplateManagerHandle}; use tari_validator_node_rpc::client::{TariValidatorNodeRpcClientFactory, ValidatorNodeClientFactory}; @@ -309,7 +309,7 @@ impl EventScanner { Ok(()) } - fn store_substates_in_db(&self, updates: &[SubstateUpdate], timestamp: u64) -> Result<(), anyhow::Error> { + fn store_substates_in_db(&self, updates: &[SubstateUpdateProof], timestamp: u64) -> Result<(), anyhow::Error> { let mut tx = self.substate_store.create_write_tx()?; // store/update up substates if any for create in updates.iter().filter_map(|up| up.as_create()) { @@ -527,7 +527,7 @@ impl EventScanner { let update = msg .into_substate_update() .ok_or_else(|| anyhow::anyhow!("Expected a substate"))?; - let update = SubstateUpdate::try_from(update)?; + let update = SubstateUpdateProof::try_from(update)?; diff.push(update); } diff --git a/applications/tari_validator_node/Cargo.toml b/applications/tari_validator_node/Cargo.toml index 4a58c1edbb..33ada6e52e 100644 --- a/applications/tari_validator_node/Cargo.toml +++ b/applications/tari_validator_node/Cargo.toml @@ -46,6 +46,7 @@ tari_networking = { workspace = true } tari_rpc_framework = { workspace = true } tari_template_builtin = { workspace = true } tari_swarm = { workspace = true } +tari_state_tree = { workspace = true } tari_sidechain = { workspace = true } sqlite_message_logger = { workspace = true } diff --git a/applications/tari_validator_node/src/bootstrap.rs b/applications/tari_validator_node/src/bootstrap.rs index ebc191638f..2f34771f84 100644 --- a/applications/tari_validator_node/src/bootstrap.rs +++ b/applications/tari_validator_node/src/bootstrap.rs @@ -219,16 +219,9 @@ pub async fn spawn_services( info!(target: LOG_TARGET, "State store initializing"); - let sidechain_id = config - .validator_node - .validator_node_sidechain_id - .as_ref() - .map(|pk| pk.to_byte_type()); - let state_store = ValidatorNodeStateStore::open(&config.validator_node.state_db_path, DatabaseOptions::default())?; - state_store - .with_write_tx(|tx| bootstrap_state(tx, config.network, consensus_constants.num_preshards, sidechain_id))?; + state_store.with_write_tx(|tx| bootstrap_state(tx, config.network, consensus_constants.num_preshards))?; info!(target: LOG_TARGET, "Epoch manager initializing"); let epoch_manager_config = EpochManagerConfig { @@ -335,6 +328,12 @@ pub async fn spawn_services( #[cfg(not(feature = "metrics"))] let metrics = NoopHooks; + let sidechain_id = config + .validator_node + .validator_node_sidechain_id + .as_ref() + .map(|pk| pk.to_byte_type()); + let signing_service = consensus::TariSignatureService::new(keypair.clone()); let (consensus_join_handle, consensus_handle) = consensus::spawn( config.network, diff --git a/applications/tari_validator_node/src/p2p/rpc/block_sync_task.rs b/applications/tari_validator_node/src/p2p/rpc/block_sync_task.rs index 72b955af8e..029fe863c6 100644 --- a/applications/tari_validator_node/src/p2p/rpc/block_sync_task.rs +++ b/applications/tari_validator_node/src/p2p/rpc/block_sync_task.rs @@ -12,7 +12,7 @@ use tari_ootle_p2p::{ proto::rpc::{sync_blocks_response::SyncData, QuorumCertificates, SyncBlocksResponse}, }; use tari_ootle_storage::{ - consensus_models::{Block, SubstateCreatedProof, SubstateUpdate, TransactionRecord}, + consensus_models::{Block, SubstateCreatedProof, SubstateUpdateProof, TransactionRecord}, StateStore, StateStoreReadTransaction, StorageError, @@ -27,7 +27,7 @@ const BLOCK_BUFFER_SIZE: usize = 15; struct BlockData { block: Block, qcs: Vec, - substates: Vec, + substates: Vec, transactions: Vec, transaction_receipts: Vec, } diff --git a/applications/tari_validator_node/src/p2p/rpc/service_impl.rs b/applications/tari_validator_node/src/p2p/rpc/service_impl.rs index 923268685b..8ad9fed70c 100644 --- a/applications/tari_validator_node/src/p2p/rpc/service_impl.rs +++ b/applications/tari_validator_node/src/p2p/rpc/service_impl.rs @@ -26,7 +26,16 @@ use log::*; use tari_bor::encode; use tari_consensus_types::BlockId; use tari_epoch_manager::{service::EpochManagerHandle, EpochManagerReader}; -use tari_ootle_common_types::{optional::Optional, shard::Shard, Epoch, NodeHeight, PeerAddress, SubstateRequirement}; +use tari_ootle_common_types::{ + displayable::Displayable, + optional::Optional, + shard::Shard, + Epoch, + NodeHeight, + NumPreshards, + PeerAddress, + SubstateRequirement, +}; use tari_ootle_p2p::{ proto, proto::rpc::{ @@ -47,7 +56,7 @@ use tari_ootle_p2p::{ }, }; use tari_ootle_storage::{ - consensus_models::{Block, EpochCheckpoint, StateTransitionId, SubstateRecord, TransactionRecord}, + consensus_models::{Block, EpochCheckpoint, SubstateRecord, TransactionRecord}, StateStore, }; use tari_rpc_framework::{Request, Response, RpcStatus, Streaming}; @@ -171,27 +180,14 @@ impl ValidatorNodeRpcSe })); }; - let created_qc = substate - .get_created_proposal_certificate(&tx) - // TODO: We may not have this... We dont sync PCs. Hmm... - // This does not actually prove this substate was committed anyhow. - .optional() - .map_err(RpcStatus::log_internal_error(LOG_TARGET))?; - - let resp = if substate.is_destroyed() { - let destroyed_qc = substate - .get_destroyed_proposal_certificate(&tx) - .map_err(RpcStatus::log_internal_error(LOG_TARGET))?; + let resp = if let Some(destroyed) = substate.destroyed() { GetSubstateResponse { status: SubstateStatus::Down as i32, address: substate.substate_id().to_bytes(), + substate: vec![], version: substate.version(), - quorum_certificates: created_qc - .into_iter() - .chain(destroyed_qc) - .map(|qc| (&qc).into()) - .collect(), - ..Default::default() + created_at_state_version: substate.created().at_state_version, + destroyed_at_state_version: destroyed.at_state_version, } } else { GetSubstateResponse { @@ -202,7 +198,8 @@ impl ValidatorNodeRpcSe .substate_value() .map(|v| v.to_bytes()) .ok_or_else(|| RpcStatus::general("NEVER HAPPEN: UP substate has no value"))?, - quorum_certificates: created_qc.iter().map(Into::into).collect(), + created_at_state_version: substate.created().at_state_version, + destroyed_at_state_version: Default::default(), } }; @@ -380,20 +377,31 @@ impl ValidatorNodeRpcSe let (sender, receiver) = mpsc::channel(10); - let start_epoch = Epoch(req.start_epoch); - let start_shard = Shard::from(req.start_shard); - let last_state_transition_for_chain = StateTransitionId::new(start_epoch, start_shard, req.start_seq); + let shard = Shard::from_u32(req.shard); + if shard > NumPreshards::MAX_SHARD { + return Err(RpcStatus::bad_request(format!( + "Shard {} out of range. Maximum shard is {}", + shard, + NumPreshards::MAX_SHARD + ))); + } - let end_epoch = Epoch(req.current_epoch); - info!(target: LOG_TARGET, "🌍peer initiated sync with this node ({}, {}, seq={}) to {}", start_epoch, start_shard, req.start_seq, end_epoch); + let end_epoch = Some(req.until_epoch).filter(|e| *e > 0).map(Epoch::from); + info!(target: LOG_TARGET, "🌍peer initiated sync with this node (start: v{}, {}) to {}", req.start_state_version, shard, end_epoch.display()); + if req.start_state_version == 0 { + return Err(RpcStatus::bad_request("start_state_version must be greater than 0")); + } task::spawn( StateSyncTask::new( self.state_store.clone(), sender, - last_state_transition_for_chain, + shard, + req.start_state_version, end_epoch, - STATE_SYNC_MAX_BATCH_SIZE, + STATE_SYNC_MAX_BATCH_SIZE + .try_into() + .expect("STATE_SYNC_MAX_BATCH_SIZE is not zero"), ) .run(), ); diff --git a/applications/tari_validator_node/src/p2p/rpc/state_sync_task.rs b/applications/tari_validator_node/src/p2p/rpc/state_sync_task.rs index ffe068181a..b52b3c4f13 100644 --- a/applications/tari_validator_node/src/p2p/rpc/state_sync_task.rs +++ b/applications/tari_validator_node/src/p2p/rpc/state_sync_task.rs @@ -1,64 +1,79 @@ // Copyright 2023 The Tari Project // SPDX-License-Identifier: BSD-3-Clause +use std::num::NonZeroUsize; + use log::*; -use tari_ootle_common_types::{optional::Optional, Epoch}; +use tari_ootle_common_types::{optional::Optional, shard::Shard, Epoch}; use tari_ootle_p2p::proto::rpc::SyncStateResponse; use tari_ootle_storage::{ - consensus_models::{StateTransition, StateTransitionId}, + consensus_models::{StateTransition, StateVersionTransitions}, StateStore, StorageError, }; use tari_rpc_framework::RpcStatus; +use tari_state_tree::Version; use tokio::sync::mpsc; const LOG_TARGET: &str = "tari::ootle::rpc::sync_task"; -type UpdateBuffer = Vec; - pub struct StateSyncTask { store: TStateStore, sender: mpsc::Sender>, - start_state_transition_id: StateTransitionId, - current_epoch: Epoch, - batch_size: usize, + shard: Shard, + start_state_version: Version, + end_epoch: Option, + batch_size: NonZeroUsize, } impl StateSyncTask { pub fn new( store: TStateStore, sender: mpsc::Sender>, - start_state_transition_id: StateTransitionId, - current_epoch: Epoch, - batch_size: usize, + shard: Shard, + start_state_version: Version, + end_epoch: Option, + batch_size: NonZeroUsize, ) -> Self { Self { store, sender, - start_state_transition_id, - current_epoch, + shard, + start_state_version, + end_epoch, batch_size, } } pub async fn run(mut self) -> Result<(), ()> { - let mut buffer = Vec::with_capacity(self.batch_size); - let mut current_state_transition_id = self.start_state_transition_id; + let mut current_state_version = self.start_state_version; let mut counter = 0usize; loop { - match self.fetch_next_batch(&mut buffer, current_state_transition_id) { - Ok(Some(last_state_transition_id)) => { - info!(target: LOG_TARGET, "🌍Fetched {} state transitions up to transition {}", buffer.len(), last_state_transition_id); - current_state_transition_id = last_state_transition_id; + match self.fetch_next_batch(current_state_version) { + Ok(Some(transitions)) => { + info!(target: LOG_TARGET, "🌍 Fetched {} state transition(s) up to v{}", transitions.updates.len(), transitions.state_version); + if let Some(end_epoch) = self.end_epoch { + // TODO(perf): might be better to not load in the first place, however also might incur the cost + // of a db index or loading from db anyway + if transitions.epoch > end_epoch { + info!(target: LOG_TARGET, "🌍 Reached end of requested epoch: {}", end_epoch); + return Ok(()); + } + } + + current_state_version = transitions.state_version + 1; + counter += transitions.updates.len(); + + self.send_responses(transitions).await?; }, Ok(None) => { // TODO: differentiate between not found and end of stream // self.send(Err(RpcStatus::not_found(format!( - // "State transition not found with id={current_state_transition_id}" + // "State transition not found with id={current_state_version}" // )))) // .await?; - info!(target: LOG_TARGET, "🌍sync complete ({}). {} update(s) sent.", current_state_transition_id, counter); + info!(target: LOG_TARGET, "🌍sync complete ({}). {} update(s) sent.", current_state_version, counter); // Finished return Ok(()); }, @@ -67,46 +82,18 @@ impl StateSyncTask { return Err(()); }, } - - let num_items = buffer.len(); - debug!( - target: LOG_TARGET, - "Sending {num_items} state updates to peer. Current transition id: {current_state_transition_id}", - ); - - counter += buffer.len(); - self.send_state_transitions(buffer.drain(..)).await?; - - // If we didn't fill up the buffer, so we're done - if num_items < buffer.capacity() { - debug!( target: LOG_TARGET, "Sync to last commit complete. Streamed {} item(s)", counter); - break; - } } - - Ok(()) } fn fetch_next_batch( &self, - buffer: &mut UpdateBuffer, - current_state_transition_id: StateTransitionId, - ) -> Result, StorageError> { - self.store.with_read_tx(|tx| { - let state_transitions = - StateTransition::get_n_after(tx, self.batch_size, current_state_transition_id, self.current_epoch) - .optional()? - .unwrap_or_default(); - - let Some(last) = state_transitions.last() else { - return Ok(None); - }; - - let last_state_transition_id = last.id; - buffer.extend(state_transitions); - - Ok::<_, StorageError>(Some(last_state_transition_id)) - }) + current_state_version: Version, + ) -> Result, StorageError> { + let transitions = self.store.with_read_tx(|tx| { + // TODO: make it optional for the client to request values + StateTransition::get_for_shard(tx, self.shard, current_state_version, true).optional() + })?; + Ok(transitions) } async fn send(&mut self, result: Result) -> Result<(), ()> { @@ -120,14 +107,21 @@ impl StateSyncTask { Ok(()) } - async fn send_state_transitions>( - &mut self, - state_transitions: I, - ) -> Result<(), ()> { - self.send(Ok(SyncStateResponse { - transitions: state_transitions.into_iter().map(Into::into).collect(), - })) - .await?; + async fn send_responses(&mut self, transitions: StateVersionTransitions) -> Result<(), ()> { + let chunks = transitions.into_chunks(self.batch_size.get()); + let num_chunks = chunks.len(); + + for (i, chunk) in chunks.into_iter().enumerate() { + let updates = chunk.updates.into_iter().map(Into::into).collect(); + + self.send(Ok(SyncStateResponse { + state_version: chunk.state_version, + updates, + has_more: i < num_chunks - 1, + epoch: Some(chunk.epoch.into()), + })) + .await?; + } Ok(()) } diff --git a/applications/tari_validator_node/src/state_bootstrap.rs b/applications/tari_validator_node/src/state_bootstrap.rs index 3aa82e4717..a62c2eddd5 100644 --- a/applications/tari_validator_node/src/state_bootstrap.rs +++ b/applications/tari_validator_node/src/state_bootstrap.rs @@ -5,8 +5,6 @@ use std::ops::Deref; use serde::Serialize; use tari_bor::cbor; -use tari_common_types::types::FixedHash; -use tari_consensus_types::BlockId; use tari_engine_types::{ component::{ComponentBody, ComponentHeader}, resource::Resource, @@ -19,16 +17,16 @@ use tari_ootle_common_types::{ Network, NodeAddressable, NumPreshards, - ShardGroup, - ToSubstateAddress, VersionedSubstateId, + VersionedSubstateIdRef, }; use tari_ootle_storage::{ - consensus_models::{Block, SubstateRecord}, + consensus_models::{SubstateRecord, SubstateTransition, SubstateUpdateBatch}, StateStoreReadTransaction, StateStoreWriteTransaction, StorageError, }; +use tari_state_tree::Version; use tari_template_lib::{ auth::{ComponentAccessRules, OwnerRule, ResourceAccessRules}, constants::{ @@ -40,12 +38,14 @@ use tari_template_lib::{ XTR_FAUCET_VAULT_ADDRESS, }, models::Metadata, - prelude::{ResourceType, RistrettoPublicKeyBytes}, + prelude::ResourceType, resource::TOKEN_SYMBOL, rule, types::EntityId, }; +const INITIAL_STATE_VERSION: Version = 0; + pub fn has_bootstrapped(tx: &TTx) -> Result { // Assume that if the public identity resource exists, then the rest of the state has been bootstrapped SubstateRecord::exists( @@ -54,12 +54,7 @@ pub fn has_bootstrapped(tx: &TTx) -> Result( - tx: &mut TTx, - network: Network, - num_preshards: NumPreshards, - sidechain_id: Option, -) -> Result<(), StorageError> +pub fn bootstrap_state(tx: &mut TTx, network: Network, num_preshards: NumPreshards) -> Result<(), StorageError> where TTx: StateStoreWriteTransaction + Deref, TTx::Target: StateStoreReadTransaction, @@ -80,14 +75,7 @@ where 0, false, ); - create_substate( - tx, - network, - num_preshards, - sidechain_id, - PUBLIC_IDENTITY_RESOURCE_ADDRESS, - value, - )?; + create_substate(tx, num_preshards, PUBLIC_IDENTITY_RESOURCE_ADDRESS, value)?; let is_testnet = !matches!(network, Network::MainNet); let symbol = if is_testnet { "tXTR" } else { "XTR" }; @@ -105,29 +93,17 @@ where if is_testnet { // Create tXTR faucet - create_xtr_faucet(tx, network, num_preshards, sidechain_id)?; + create_xtr_faucet(tx, num_preshards)?; // Create NFT faucet - create_nft_faucet(tx, network, num_preshards, sidechain_id)?; + create_nft_faucet(tx, num_preshards)?; } - create_substate( - tx, - network, - num_preshards, - sidechain_id, - CONFIDENTIAL_TARI_RESOURCE_ADDRESS, - xtr_resource, - )?; + create_substate(tx, num_preshards, CONFIDENTIAL_TARI_RESOURCE_ADDRESS, xtr_resource)?; Ok(()) } -fn create_xtr_faucet( - tx: &mut TTx, - network: Network, - num_preshards: NumPreshards, - sidechain_id: Option, -) -> Result<(), StorageError> +fn create_xtr_faucet(tx: &mut TTx, num_preshards: NumPreshards) -> Result<(), StorageError> where TTx: StateStoreWriteTransaction + Deref, TTx::Target: StateStoreReadTransaction, @@ -144,14 +120,7 @@ where state: cbor!({"vault" => XTR_FAUCET_VAULT_ADDRESS}).unwrap(), }, }; - create_substate( - tx, - network, - num_preshards, - sidechain_id, - XTR_FAUCET_COMPONENT_ADDRESS, - value, - )?; + create_substate(tx, num_preshards, XTR_FAUCET_COMPONENT_ADDRESS, value)?; let value = Vault::new(ResourceContainer::Confidential { address: CONFIDENTIAL_TARI_RESOURCE_ADDRESS, @@ -162,23 +131,11 @@ where locked_revealed_amount: Default::default(), }); - create_substate( - tx, - network, - num_preshards, - sidechain_id, - XTR_FAUCET_VAULT_ADDRESS, - value, - )?; + create_substate(tx, num_preshards, XTR_FAUCET_VAULT_ADDRESS, value)?; Ok(()) } -fn create_nft_faucet( - tx: &mut TTx, - network: Network, - num_preshards: NumPreshards, - sidechain_id: Option, -) -> Result<(), StorageError> +fn create_nft_faucet(tx: &mut TTx, num_preshards: NumPreshards) -> Result<(), StorageError> where TTx: StateStoreWriteTransaction + Deref, TTx::Target: StateStoreReadTransaction, @@ -195,14 +152,7 @@ where state: cbor!({"serial_number" => 0u64}).unwrap(), }, }; - create_substate( - tx, - network, - num_preshards, - sidechain_id, - NFT_FAUCET_COMPONENT_ADDRESS, - value, - )?; + create_substate(tx, num_preshards, NFT_FAUCET_COMPONENT_ADDRESS, value)?; let metadata = Metadata::from([("name", "NFT Faucet"), (TOKEN_SYMBOL, "TNFT")]); @@ -219,22 +169,13 @@ where true, ); - create_substate( - tx, - network, - num_preshards, - sidechain_id, - NFT_FAUCET_RESOURCE_ADDRESS, - value, - )?; + create_substate(tx, num_preshards, NFT_FAUCET_RESOURCE_ADDRESS, value)?; Ok(()) } fn create_substate( tx: &mut TTx, - network: Network, num_preshards: NumPreshards, - sidechain_id: Option, substate_id: TId, value: TVal, ) -> Result<(), StorageError> @@ -245,29 +186,16 @@ where TId: Into, TVal: Into, { - let genesis_block = Block::genesis( - network, - Epoch(0), - FixedHash::zero(), - ShardGroup::all_shards(num_preshards), - FixedHash::default(), - sidechain_id, - ); let substate_id = substate_id.into(); - let id = VersionedSubstateId::new(substate_id, 0); - let shard = id.to_substate_address().to_shard(num_preshards); - SubstateRecord { - version: id.version(), - substate_id: id.into_substate_id(), - substate_value: Some(value.into()), - state_hash: Default::default(), - created_justify: genesis_block.justify().calculate_id(), - created_block: BlockId::zero(), - created_by_shard: shard, - created_at_epoch: Epoch(0), - destroyed: None, - } - .create(tx)?; + let shard = VersionedSubstateIdRef::new(&substate_id, 0).to_shard(num_preshards); + let mut batch = SubstateUpdateBatch::new(Epoch::zero()); + batch.add_transition(shard, INITIAL_STATE_VERSION, SubstateTransition::Up { + id: substate_id, + version: 0, + substate_or_hash: value.into().into(), + }); + + SubstateRecord::commit_batch(tx, batch)?; Ok(()) } diff --git a/crates/common_types/Cargo.toml b/crates/common_types/Cargo.toml index 96aa0b61d3..cc268b2617 100644 --- a/crates/common_types/Cargo.toml +++ b/crates/common_types/Cargo.toml @@ -19,6 +19,7 @@ libp2p-identity = { workspace = true, features = ["sr25519", "serde", "peerid"] borsh = { workspace = true } blake2 = { workspace = true } +bounded-vec = { workspace = true, features = ["serde"] } digest = { workspace = true } ethnum = { workspace = true } newtype-ops = { workspace = true } diff --git a/crates/common_types/src/committee.rs b/crates/common_types/src/committee.rs index ea515ac900..4ddaf268e1 100644 --- a/crates/common_types/src/committee.rs +++ b/crates/common_types/src/committee.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; use tari_engine_types::substate::SubstateId; use tari_template_lib_types::crypto::RistrettoPublicKeyBytes; -use crate::{Epoch, NumPreshards, ShardGroup, SubstateAddress, VotePower}; +use crate::{Epoch, NumPreshards, ShardGroup, SubstateAddress, VersionedSubstateIdRef, VotePower}; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Default, Hash)] #[cfg_attr( @@ -315,7 +315,7 @@ impl CommitteeInfo { return true; } // version doesnt affect shard - let addr = SubstateAddress::from_substate_id(substate_id, 0); + let addr = VersionedSubstateIdRef::new(substate_id, 0); let shard = addr.to_shard(self.num_shards); self.shard_group.contains(&shard) } diff --git a/crates/common_types/src/lib.rs b/crates/common_types/src/lib.rs index 5b958245df..8b83be093f 100644 --- a/crates/common_types/src/lib.rs +++ b/crates/common_types/src/lib.rs @@ -67,3 +67,5 @@ mod vote_power; pub use vote_power::*; pub mod base_layer_hashing; +mod shard_state_versions; +pub use shard_state_versions::*; diff --git a/crates/common_types/src/num_preshards.rs b/crates/common_types/src/num_preshards.rs index 01eb5177e2..22ccb6c588 100644 --- a/crates/common_types/src/num_preshards.rs +++ b/crates/common_types/src/num_preshards.rs @@ -27,6 +27,7 @@ pub enum NumPreshards { impl NumPreshards { pub const MAX: Self = Self::P256; + pub const MAX_SHARD: Shard = Shard::from_u32(Self::MAX.as_u32()); pub const fn as_u32(self) -> u32 { self as u32 diff --git a/crates/common_types/src/shard.rs b/crates/common_types/src/shard.rs index 8b45fb7509..f198192459 100644 --- a/crates/common_types/src/shard.rs +++ b/crates/common_types/src/shard.rs @@ -6,7 +6,7 @@ use std::{fmt::Display, ops::RangeInclusive}; use borsh::BorshSerialize; use serde::{Deserialize, Serialize}; -use crate::{uint::U256, NumPreshards, SubstateAddress}; +use crate::{uint::U256, NumPreshards, ShardGroup, SubstateAddress}; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, BorshSerialize)] #[cfg_attr( @@ -18,6 +18,10 @@ use crate::{uint::U256, NumPreshards, SubstateAddress}; pub struct Shard(#[cfg_attr(feature = "ts", ts(type = "number"))] u32); impl Shard { + pub const fn from_u32(v: u32) -> Self { + Self(v) + } + /// Returns the first available shard in the whole range. /// Note: it starts from `1` as `0` is reserved for global substates. pub const fn first() -> Shard { @@ -42,6 +46,15 @@ impl Shard { self.0 } + pub fn relative_to_shard_group_start(self, shard_group: ShardGroup) -> Option { + if !shard_group.contains(&self) { + // Also return None for global + return None; + } + let relative_index = self.as_u32().checked_sub(shard_group.start().as_u32())?; + Some(relative_index as usize) + } + pub fn to_substate_address_range(self, num_shards: NumPreshards) -> RangeInclusive { if num_shards.is_one() || self.is_global() { return RangeInclusive::new(SubstateAddress::zero(), SubstateAddress::max()); diff --git a/crates/common_types/src/shard_group.rs b/crates/common_types/src/shard_group.rs index 048da42eea..5508fe5bd5 100644 --- a/crates/common_types/src/shard_group.rs +++ b/crates/common_types/src/shard_group.rs @@ -109,6 +109,10 @@ impl ShardGroup { }) } + pub fn shard_iter_with_global(self) -> impl Iterator + 'static { + iter::once(Shard::global()).chain(self.shard_iter()) + } + /// Returns the intersection of two shard groups, if they overlap. pub fn intersection(&self, other: &ShardGroup) -> Option { if self.overlaps_shard_group(other) { @@ -129,7 +133,7 @@ impl ShardGroup { } pub fn contains(&self, shard: &Shard) -> bool { - self.as_range().contains(shard) + self.as_range_inclusive().contains(shard) } pub fn contains_or_global(&self, shard: &Shard) -> bool { @@ -143,7 +147,7 @@ impl ShardGroup { self.start <= other.end_inclusive && self.end_inclusive >= other.start } - pub fn as_range(&self) -> RangeInclusive { + pub const fn as_range_inclusive(&self) -> RangeInclusive { self.start..=self.end_inclusive } diff --git a/crates/common_types/src/shard_state_versions.rs b/crates/common_types/src/shard_state_versions.rs new file mode 100644 index 0000000000..117550314c --- /dev/null +++ b/crates/common_types/src/shard_state_versions.rs @@ -0,0 +1,180 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use bounded_vec::BoundedVec; +pub use bounded_vec::BoundedVecOutOfBounds; +use tari_bor::{Deserialize, Serialize}; + +use crate::{shard::Shard, NumPreshards, ShardGroup}; + +/// Maximum number of shards is one more than the maximum number of presharding options to allow for the global shard +const MAX_SHARDS: usize = NumPreshards::MAX_SHARD.as_u32() as usize + 1; + +type BoundedVersionVec = BoundedVec; + +/// The state versions for each shard that maps each shard managed by the ShardGroup (including the +/// global shard) to a state version. +/// +/// For example, if the ShardGroup is [1, 3], the state versions will contain 4 +/// elements. The first element is always the global shard (shard 0) version. The second element is the state +/// version for shard 1, third is shard 2, and forth is shard 3. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr( + feature = "ts", + derive(ts_rs::TS), + ts(export, export_to = "../../bindings/src/types/") +)] +#[serde(transparent)] +pub struct ShardStateVersions { + #[cfg_attr(feature = "ts", ts(type = "number[]"))] + inner: BoundedVersionVec, +} + +impl ShardStateVersions { + pub fn genesis(shard_group: ShardGroup) -> Self { + Self { + inner: BoundedVersionVec::try_from(vec![0; shard_group.len() + 1]) + .expect("Empty vec should always be valid"), + } + } + + pub fn from_vec(shard_versions: Vec) -> Result { + Ok(Self { + inner: BoundedVersionVec::from_vec(shard_versions)?, + }) + } + + pub fn into_vec(self) -> Vec { + self.inner.into() + } + + pub fn get(&self, shard_index: usize) -> Option { + self.inner.get(shard_index).copied() + } + + pub fn get_global(&self) -> u64 { + *self.inner.first() + } + + pub fn shard_to_index(shard_group: ShardGroup, shard: Shard) -> Option { + if shard.is_global() { + return Some(0); + } + + if !shard_group.contains_or_global(&shard) { + return None; + } + shard_group.checked_len()?; + if shard_group.end().as_u32() as usize > MAX_SHARDS { + return None; + } + let index = shard.as_u32().checked_sub(shard_group.start().as_u32())? as usize; + // + 1 to account for the global shard at index 0 + Some(index + 1) + } + + pub fn get_by_shard_checked(&self, shard_group: ShardGroup, shard: Shard) -> Option { + let index = Self::shard_to_index(shard_group, shard)?; + self.inner.get(index).copied() + } + + pub fn len(&self) -> usize { + self.inner.as_slice().len() + } + + pub fn is_empty(&self) -> bool { + false + } + + pub fn as_slice(&self) -> &[u64] { + self.inner.as_slice() + } + + pub fn apply_bitmap(mut self, bitmap: Vec) -> Self { + if self.len() != bitmap.len() { + panic!("Length mismatch: expected {} but got {}", self.len(), bitmap.len()); + } + + let inner_mut: &mut [u64] = self.inner.as_mut(); + for (i, _) in bitmap.into_iter().enumerate().filter(|(_, v)| *v) { + inner_mut[i] += 1; + } + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_gets_by_index() { + let versions = ShardStateVersions::from_vec(vec![1, 2, 3]).unwrap(); + assert_eq!(versions.get(0), Some(1)); + assert_eq!(versions.get(1), Some(2)); + assert_eq!(versions.get(2), Some(3)); + assert_eq!(versions.get(3), None); + assert_eq!(versions.len(), 3); + assert!(!versions.is_empty()); + } + + #[test] + fn it_gets_by_shard() { + let v = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]; + let num_non_global_shards = v.len() as u32 - 1; + let versions = ShardStateVersions::from_vec(v).unwrap(); + let shard_group = ShardGroup::new(100, 100 + num_non_global_shards); + assert_eq!(shard_group.len(), versions.len()); + let v = versions.get_by_shard_checked(shard_group, 0.into()).unwrap(); + assert_eq!(v, 1, "returned incorrect version {v} for global shard"); + + for (i, shard) in (100..100 + num_non_global_shards).enumerate() { + let v = versions + .get_by_shard_checked(shard_group, shard.into()) + .unwrap_or_else(|| panic!("Shard {} not found", shard)); + assert_eq!(v, i as u64 + 2, "returned incorrect version {v} for shard {}", shard); + } + let v = versions.get_by_shard_checked(shard_group, Shard::from(13)); + assert!(v.is_none()); + } + + #[test] + fn it_errors_if_more_then_max_shards() { + let e = ShardStateVersions::from_vec(vec![1; MAX_SHARDS + 1]).unwrap_err(); + assert!(matches!(e, BoundedVecOutOfBounds::UpperBoundError { .. })); + } + + #[test] + fn it_deserializes_if_serialized_vec_is_within_bounds() { + let v = vec![1, 2, 3]; + let serialized = tari_bor::encode(&v).unwrap(); + let deserialized: ShardStateVersions = tari_bor::decode(&serialized).unwrap(); + assert_eq!(deserialized.as_slice(), v); + } + + #[test] + fn it_errors_if_serialized_vec_is_empty() { + let v = Vec::::new(); + let serialized = tari_bor::encode(&v).unwrap(); + tari_bor::decode::(&serialized).unwrap_err(); + } + + #[test] + fn it_errors_if_serialized_vec_is_too_large() { + let v = vec![1; MAX_SHARDS + 1]; + let serialized = tari_bor::encode(&v).unwrap(); + tari_bor::decode::(&serialized).unwrap_err(); + } + + #[test] + fn it_applies_a_bitmap_to_increment_versions() { + let versions = ShardStateVersions::from_vec(vec![1, 2, 3]).unwrap(); + let bitmap = vec![true, false, true]; + let updated_versions = versions.apply_bitmap(bitmap); + + assert_eq!(updated_versions.get(0), Some(2)); + assert_eq!(updated_versions.get(1), Some(2)); + assert_eq!(updated_versions.get(2), Some(4)); + assert_eq!(updated_versions.len(), 3); + } +} diff --git a/crates/common_types/src/substate_address.rs b/crates/common_types/src/substate_address.rs index 0c2efaff6a..bcb67123df 100644 --- a/crates/common_types/src/substate_address.rs +++ b/crates/common_types/src/substate_address.rs @@ -550,56 +550,56 @@ mod tests { #[test] fn it_returns_the_correct_shard_group() { let group = SubstateAddress::zero().to_shard_group(NumPreshards::P4, 2); - assert_eq!(group.as_range(), Shard::from(1)..=Shard::from(2)); + assert_eq!(group.as_range_inclusive(), Shard::from(1)..=Shard::from(2)); let group = plus_one(address_at(0, 4)).to_shard_group(NumPreshards::P4, 2); - assert_eq!(group.as_range(), Shard::from(1)..=Shard::from(2)); + assert_eq!(group.as_range_inclusive(), Shard::from(1)..=Shard::from(2)); let group = address_at(1, 4).to_shard_group(NumPreshards::P4, 2); - assert_eq!(group.as_range(), Shard::from(1)..=Shard::from(2)); + assert_eq!(group.as_range_inclusive(), Shard::from(1)..=Shard::from(2)); let group = address_at(2, 4).to_shard_group(NumPreshards::P4, 2); - assert_eq!(group.as_range(), Shard::from(3)..=Shard::from(4)); + assert_eq!(group.as_range_inclusive(), Shard::from(3)..=Shard::from(4)); let group = address_at(3, 4).to_shard_group(NumPreshards::P4, 2); - assert_eq!(group.as_range(), Shard::from(3)..=Shard::from(4)); + assert_eq!(group.as_range_inclusive(), Shard::from(3)..=Shard::from(4)); let group = SubstateAddress::max().to_shard_group(NumPreshards::P4, 2); - assert_eq!(group.as_range(), Shard::from(3)..=Shard::from(4)); + assert_eq!(group.as_range_inclusive(), Shard::from(3)..=Shard::from(4)); let group = minus_one(address_at(1, 64)).to_shard_group(NumPreshards::P64, 16); - assert_eq!(group.as_range(), Shard::from(1)..=Shard::from(4)); + assert_eq!(group.as_range_inclusive(), Shard::from(1)..=Shard::from(4)); let group = address_at(4, 64).to_shard_group(NumPreshards::P64, 16); - assert_eq!(group.as_range(), Shard::from(5)..=Shard::from(8)); + assert_eq!(group.as_range_inclusive(), Shard::from(5)..=Shard::from(8)); let group = address_at(8, 64).to_shard_group(NumPreshards::P64, 2); - assert_eq!(group.as_range(), Shard::from(1)..=Shard::from(32)); + assert_eq!(group.as_range_inclusive(), Shard::from(1)..=Shard::from(32)); let group = address_at(5, 8).to_shard_group(NumPreshards::P64, 2); - assert_eq!(group.as_range(), Shard::from(33)..=Shard::from(64)); + assert_eq!(group.as_range_inclusive(), Shard::from(33)..=Shard::from(64)); // On boundary let group = address_at(0, 8).to_shard_group(NumPreshards::P64, 2); - assert_eq!(group.as_range(), Shard::from(1)..=Shard::from(32)); + assert_eq!(group.as_range_inclusive(), Shard::from(1)..=Shard::from(32)); let group = address_at(4, 8).to_shard_group(NumPreshards::P64, 2); - assert_eq!(group.as_range(), Shard::from(33)..=Shard::from(64)); + assert_eq!(group.as_range_inclusive(), Shard::from(33)..=Shard::from(64)); let group = address_at(8, 8).to_shard_group(NumPreshards::P64, 2); - assert_eq!(group.as_range(), Shard::from(33)..=Shard::from(64)); + assert_eq!(group.as_range_inclusive(), Shard::from(33)..=Shard::from(64)); let group = plus_one(address_at(3, 64)).to_shard_group(NumPreshards::P64, 32); - assert_eq!(group.as_range(), Shard::from(3)..=Shard::from(4)); + assert_eq!(group.as_range_inclusive(), Shard::from(3)..=Shard::from(4)); let group = plus_one(address_at(3, 64)).to_shard_group(NumPreshards::P64, 32); - assert_eq!(group.as_range(), Shard::from(3)..=Shard::from(4)); + assert_eq!(group.as_range_inclusive(), Shard::from(3)..=Shard::from(4)); let group = address_at(16, 64).to_shard_group(NumPreshards::P64, 32); - assert_eq!(group.as_range(), Shard::from(17)..=Shard::from(18)); + assert_eq!(group.as_range_inclusive(), Shard::from(17)..=Shard::from(18)); let group = minus_one(address_at(1, 4)).to_shard_group(NumPreshards::P64, 64); - assert_eq!(group.as_range(), Shard::from(17)..=Shard::from(17)); + assert_eq!(group.as_range_inclusive(), Shard::from(17)..=Shard::from(17)); let group = address_at(66, 256).to_shard_group(NumPreshards::P64, 16); - assert_eq!(group.as_range(), Shard::from(17)..=Shard::from(20)); + assert_eq!(group.as_range_inclusive(), Shard::from(17)..=Shard::from(20)); } #[test] @@ -615,14 +615,18 @@ mod tests { let group = address_at(at, num_shards.as_u32()).to_shard_group(num_shards, NUM_COMMITTEES); if at < num_shards.as_u32() / NUM_COMMITTEES { assert_eq!( - group.as_range(), + group.as_range_inclusive(), Shard::from(1)..=Shard::from(num_shards.as_u32() / NUM_COMMITTEES), "Failed at {at} for num_shards={num_shards}" ); } else { let range = Shard::from(num_shards.as_u32() / NUM_COMMITTEES + 1)..=Shard::from(num_shards.as_u32()); - assert_eq!(group.as_range(), range, "Failed at {at} for num_shards={num_shards}"); + assert_eq!( + group.as_range_inclusive(), + range, + "Failed at {at} for num_shards={num_shards}" + ); } } } @@ -650,65 +654,65 @@ mod tests { let group = address_at(0, 64).to_shard_group(NumPreshards::P64, 3); // First shard group gets an extra shard to cover the remainder - assert_eq!(group.as_range(), Shard::from(1)..=Shard::from(22)); + assert_eq!(group.as_range_inclusive(), Shard::from(1)..=Shard::from(22)); assert_eq!(group.len(), 22); let group = address_at(31, 64).to_shard_group(NumPreshards::P64, 3); - assert_eq!(group.as_range(), Shard::from(23)..=Shard::from(43)); + assert_eq!(group.as_range_inclusive(), Shard::from(23)..=Shard::from(43)); assert_eq!(group.len(), 21); let group = address_at(50, 64).to_shard_group(NumPreshards::P64, 3); - assert_eq!(group.as_range(), Shard::from(44)..=Shard::from(64)); + assert_eq!(group.as_range_inclusive(), Shard::from(44)..=Shard::from(64)); assert_eq!(group.len(), 21); let group = address_at(3, 64).to_shard_group(NumPreshards::P64, 7); - assert_eq!(group.as_range(), Shard::from(1)..=Shard::from(10)); + assert_eq!(group.as_range_inclusive(), Shard::from(1)..=Shard::from(10)); assert_eq!(group.len(), 10); let group = address_at(11, 64).to_shard_group(NumPreshards::P64, 7); - assert_eq!(group.as_range(), Shard::from(11)..=Shard::from(19)); + assert_eq!(group.as_range_inclusive(), Shard::from(11)..=Shard::from(19)); assert_eq!(group.len(), 9); let group = address_at(22, 64).to_shard_group(NumPreshards::P64, 7); - assert_eq!(group.as_range(), Shard::from(20)..=Shard::from(28)); + assert_eq!(group.as_range_inclusive(), Shard::from(20)..=Shard::from(28)); assert_eq!(group.len(), 9); let group = address_at(60, 64).to_shard_group(NumPreshards::P64, 7); - assert_eq!(group.as_range(), Shard::from(56)..=Shard::from(64)); + assert_eq!(group.as_range_inclusive(), Shard::from(56)..=Shard::from(64)); assert_eq!(group.len(), 9); let group = address_at(64, 64).to_shard_group(NumPreshards::P64, 7); - assert_eq!(group.as_range(), Shard::from(56)..=Shard::from(64)); + assert_eq!(group.as_range_inclusive(), Shard::from(56)..=Shard::from(64)); assert_eq!(group.len(), 9); let group = SubstateAddress::zero().to_shard_group(NumPreshards::P8, 3); - assert_eq!(group.as_range(), Shard::from(1)..=Shard::from(3)); + assert_eq!(group.as_range_inclusive(), Shard::from(1)..=Shard::from(3)); let group = address_at(1, 8).to_shard_group(NumPreshards::P8, 3); - assert_eq!(group.as_range(), Shard::from(1)..=Shard::from(3)); + assert_eq!(group.as_range_inclusive(), Shard::from(1)..=Shard::from(3)); let group = address_at(1, 8).to_shard_group(NumPreshards::P8, 3); - assert_eq!(group.as_range(), Shard::from(1)..=Shard::from(3)); + assert_eq!(group.as_range_inclusive(), Shard::from(1)..=Shard::from(3)); let group = address_at(3, 8).to_shard_group(NumPreshards::P8, 3); - assert_eq!(group.as_range(), Shard::from(4)..=Shard::from(6)); + assert_eq!(group.as_range_inclusive(), Shard::from(4)..=Shard::from(6)); let group = address_at(4, 8).to_shard_group(NumPreshards::P8, 3); - assert_eq!(group.as_range(), Shard::from(4)..=Shard::from(6)); + assert_eq!(group.as_range_inclusive(), Shard::from(4)..=Shard::from(6)); let group = address_at(5, 8).to_shard_group(NumPreshards::P8, 3); - assert_eq!(group.as_range(), Shard::from(4)..=Shard::from(6)); + assert_eq!(group.as_range_inclusive(), Shard::from(4)..=Shard::from(6)); // let group = address_at(6, 8).to_shard_group(NumPreshards::P8, 3); - assert_eq!(group.as_range(), Shard::from(7)..=Shard::from(8)); + assert_eq!(group.as_range_inclusive(), Shard::from(7)..=Shard::from(8)); let group = address_at(7, 8).to_shard_group(NumPreshards::P8, 3); - assert_eq!(group.as_range(), Shard::from(7)..=Shard::from(8)); + assert_eq!(group.as_range_inclusive(), Shard::from(7)..=Shard::from(8)); let group = address_at(8, 8).to_shard_group(NumPreshards::P8, 3); - assert_eq!(group.as_range(), Shard::from(7)..=Shard::from(8)); + assert_eq!(group.as_range_inclusive(), Shard::from(7)..=Shard::from(8)); // Committee = 5 let group = address_at(4, 8).to_shard_group(NumPreshards::P8, 5); - assert_eq!(group.as_range(), Shard::from(5)..=Shard::from(6)); + assert_eq!(group.as_range_inclusive(), Shard::from(5)..=Shard::from(6)); let group = address_at(7, 8).to_shard_group(NumPreshards::P8, 5); - assert_eq!(group.as_range(), Shard::from(8)..=Shard::from(8)); + assert_eq!(group.as_range_inclusive(), Shard::from(8)..=Shard::from(8)); let group = address_at(8, 8).to_shard_group(NumPreshards::P8, 5); - assert_eq!(group.as_range(), Shard::from(8)..=Shard::from(8)); + assert_eq!(group.as_range_inclusive(), Shard::from(8)..=Shard::from(8)); } } diff --git a/crates/common_types/src/versioned_substate_id.rs b/crates/common_types/src/versioned_substate_id.rs index 649d205b68..9f81983d27 100644 --- a/crates/common_types/src/versioned_substate_id.rs +++ b/crates/common_types/src/versioned_substate_id.rs @@ -76,6 +76,9 @@ impl SubstateRequirement { /// A shard is a fixed division of the 256-bit shard space. /// If the substate version is not known, None is returned. pub fn to_shard(&self, num_shards: NumPreshards) -> Option { + if self.substate_id.is_global() { + return Some(Shard::global()); + } self.to_substate_address().map(|a| a.to_shard(num_shards)) } @@ -430,6 +433,13 @@ impl<'a> VersionedSubstateIdRef<'a> { Self { substate_id, version } } + pub fn to_shard(&self, num_preshards: NumPreshards) -> Shard { + if self.substate_id.is_global() { + return Shard::global(); + } + self.to_substate_address().to_shard(num_preshards) + } + pub fn substate_id(&self) -> &SubstateId { self.substate_id } diff --git a/crates/consensus/src/hotstuff/block_change_set.rs b/crates/consensus/src/hotstuff/block_change_set.rs index 8f35b8d83a..b49ada4bf6 100644 --- a/crates/consensus/src/hotstuff/block_change_set.rs +++ b/crates/consensus/src/hotstuff/block_change_set.rs @@ -100,7 +100,7 @@ impl BlockDecision { pub struct ProposedBlockChangeSet { block: LeafBlock, quorum_decision: Option, - substate_changes: Vec, + local_substate_changes: Vec, state_tree_diffs: IndexMap, substate_locks: IndexMap>, transaction_changes: IndexMap, @@ -116,7 +116,7 @@ impl ProposedBlockChangeSet { Self { block, quorum_decision: None, - substate_changes: Vec::new(), + local_substate_changes: Vec::new(), substate_locks: IndexMap::new(), transaction_changes: IndexMap::new(), new_transactions_to_sequence: Vec::new(), @@ -145,15 +145,15 @@ impl ProposedBlockChangeSet { pub fn clear(&mut self) { self.quorum_decision = None; - self.substate_changes.clear(); - if self.substate_changes.capacity() > MEM_MAX_BLOCK_DIFF_CHANGES { + self.local_substate_changes.clear(); + if self.local_substate_changes.capacity() > MEM_MAX_BLOCK_DIFF_CHANGES { debug!( target: LOG_TARGET, "Shrinking block_diff from {} to {}", - self.substate_changes.capacity(), + self.local_substate_changes.capacity(), MEM_MAX_BLOCK_DIFF_CHANGES ); - self.substate_changes.shrink_to(MEM_MAX_BLOCK_DIFF_CHANGES); + self.local_substate_changes.shrink_to(MEM_MAX_BLOCK_DIFF_CHANGES); } self.transaction_changes.clear(); if self.transaction_changes.capacity() > MEM_MAX_TRANSACTION_CHANGE_SIZE { @@ -232,8 +232,8 @@ impl ProposedBlockChangeSet { self } - pub fn set_substate_changes(&mut self, diff: Vec) -> &mut Self { - self.substate_changes = diff; + pub fn set_block_diff_to_commit(&mut self, diff: BlockDiff) -> &mut Self { + self.local_substate_changes = diff.into_changes(); self } @@ -447,7 +447,7 @@ impl ProposedBlockChangeSet { let _timer = TraceTimer::debug(LOG_TARGET, "ProposedBlockChangeSet::save"); // Store the block diff - BlockDiff::insert(tx, &self.block.block_id, &self.substate_changes)?; + BlockDiff::insert(tx, &self.block.block_id, &self.local_substate_changes)?; // Store the tree diffs for each affected shard for (shard, diff) in &self.state_tree_diffs { @@ -542,7 +542,7 @@ impl ProposedBlockChangeSet { let _timer = TraceTimer::debug(LOG_TARGET, "ProposedBlockChangeSet::save_for_debug"); // TODO: consider persisting this data somewhere - for change in &self.substate_changes { + for change in &self.local_substate_changes { debug!(target: LOG_TARGET, "[drop] SubstateChange: {}", change); } @@ -599,8 +599,8 @@ impl Display for ProposedBlockChangeSet { Some(decision) => write!(f, " Decision: {},", decision)?, None => write!(f, " Decision: NO VOTE, ")?, } - if !self.substate_changes.is_empty() { - write!(f, " BlockDiff: {} change(s), ", self.substate_changes.len())?; + if !self.local_substate_changes.is_empty() { + write!(f, " BlockDiff: {} change(s), ", self.local_substate_changes.len())?; } if !self.state_tree_diffs.is_empty() { write!(f, " StateTreeDiff: {} change(s), ", self.state_tree_diffs.len())?; diff --git a/crates/consensus/src/hotstuff/commit_proofs.rs b/crates/consensus/src/hotstuff/commit_proofs.rs index 51bbf2ed6d..c231b77a02 100644 --- a/crates/consensus/src/hotstuff/commit_proofs.rs +++ b/crates/consensus/src/hotstuff/commit_proofs.rs @@ -291,12 +291,13 @@ mod tests { #[test] fn it_hashes_the_header_identically_to_sidechain_header() { let parent_id = seed_hash(1).into_array().into(); + let shard_group = ShardGroup::all_shards(NumPreshards::P256); let qc1 = ProposalCertificate::new( seed_hash(2), parent_id, NodeHeight(1), Epoch(1), - ShardGroup::all_shards(NumPreshards::P256), + shard_group, vec![], QuorumDecision::Accept, ); @@ -309,7 +310,7 @@ mod tests { qc1_id, NodeHeight(2), Epoch(1), - ShardGroup::all_shards(NumPreshards::P256), + shard_group, Default::default(), Default::default(), &Default::default(), diff --git a/crates/consensus/src/hotstuff/common.rs b/crates/consensus/src/hotstuff/common.rs index b505425c48..21f4554e08 100644 --- a/crates/consensus/src/hotstuff/common.rs +++ b/crates/consensus/src/hotstuff/common.rs @@ -29,6 +29,7 @@ use tari_ootle_storage::{ EpochCheckpoint, PendingShardStateTreeDiff, SubstateChange, + TreeRootSummary, ValidatorConsensusStats, }, StateStore, @@ -260,7 +261,7 @@ where let shard_group = eoe_block.shard_group(); // Fetch the state roots of the shards in the shard group - let mut shard_roots = IndexMap::with_capacity(shard_group.len() + 1); + let mut shard_tree_summary = IndexMap::with_capacity(shard_group.len() + 1); // adding global shard first if let Some(version) = tx.state_tree_versions_get_latest(Shard::global())? { @@ -270,15 +271,24 @@ where .get_root_hash(version) .map_err(|e| HotStuffError::StateTreeError(e.into()))?; - shard_roots.insert(Shard::global(), root_hash); + shard_tree_summary.insert(Shard::global(), TreeRootSummary { + root_hash, + state_version: version, + }); } else { - shard_roots.insert(Shard::global(), SPARSE_MERKLE_PLACEHOLDER_HASH); + shard_tree_summary.insert(Shard::global(), TreeRootSummary { + root_hash: SPARSE_MERKLE_PLACEHOLDER_HASH, + state_version: 0, + }); } for shard in shard_group.shard_iter() { let Some(version) = tx.state_tree_versions_get_latest(shard)? else { // At v0 there have been no state changes - shard_roots.insert(shard, SPARSE_MERKLE_PLACEHOLDER_HASH); + shard_tree_summary.insert(shard, TreeRootSummary { + root_hash: SPARSE_MERKLE_PLACEHOLDER_HASH, + state_version: 0, + }); continue; }; @@ -288,10 +298,13 @@ where .get_root_hash(version) .map_err(|e| HotStuffError::StateTreeError(e.into()))?; - shard_roots.insert(shard, root_hash); + shard_tree_summary.insert(shard, TreeRootSummary { + root_hash, + state_version: version, + }); } - let checkpoint = EpochCheckpoint::new(commit_proof, shard_roots); + let checkpoint = EpochCheckpoint::new(commit_proof, shard_tree_summary); Ok(checkpoint) } diff --git a/crates/consensus/src/hotstuff/error.rs b/crates/consensus/src/hotstuff/error.rs index f64e265326..9430974ab6 100644 --- a/crates/consensus/src/hotstuff/error.rs +++ b/crates/consensus/src/hotstuff/error.rs @@ -6,7 +6,12 @@ use tari_consensus_types::{BlockId, LeafBlock, QcId}; use tari_epoch_manager::EpochManagerError; use tari_ootle_common_types::{Epoch, NodeHeight, ShardGroup, VersionedSubstateIdError, VotePower}; use tari_ootle_storage::{ - consensus_models::{BlockError, ForeignProposalCommitProofError, TransactionPoolError}, + consensus_models::{ + BlockError, + EpochCheckpointValidationError, + ForeignProposalCommitProofError, + TransactionPoolError, + }, StorageError, }; use tari_state_tree::StateTreeError; @@ -87,6 +92,8 @@ pub enum HotStuffError { InvariantError(String), #[error("Sync error: {0}")] SyncError(anyhow::Error), + #[error("This node needs to sync with the network: {reason}")] + NeedsSync { reason: String }, #[error("Fallen behind: local={local_epoch}/{local_height}, qc={qc_epoch}/{qc_height}")] FallenBehind { local_epoch: Epoch, @@ -113,6 +120,8 @@ pub enum HotStuffError { }, #[error("Block building error: {0}")] BlockBuildingError(#[from] BlockError), + #[error("Epoch checkpoint validation error: {0}")] + EpochCheckpointValidationError(#[from] EpochCheckpointValidationError), } impl HotStuffError { diff --git a/crates/consensus/src/hotstuff/on_propose.rs b/crates/consensus/src/hotstuff/on_propose.rs index e4cfe1fa14..67cefe2918 100644 --- a/crates/consensus/src/hotstuff/on_propose.rs +++ b/crates/consensus/src/hotstuff/on_propose.rs @@ -26,7 +26,7 @@ use tari_ootle_common_types::{ Epoch, ExtraData, NodeHeight, - SubstateAddress, + VersionedSubstateIdRef, }; use tari_ootle_storage::{ consensus_models::{ @@ -450,8 +450,8 @@ where TConsensusSpec: ConsensusSpec // This relies on the UTXO commands being ordered after transaction commands for (commitment, output) in batch.burnt_utxos { let substate_id = commitment.into(); - let addr = SubstateAddress::from_substate_id(&substate_id, 0); - let shard = addr.to_shard(local_committee_info.num_preshards()); + let id = VersionedSubstateIdRef::new(&substate_id, 0); + let shard = id.to_shard(local_committee_info.num_preshards()); let change = SubstateChange::Up { id: substate_id, shard, @@ -486,7 +486,7 @@ where TConsensusSpec: ConsensusSpec tx, local_committee_info.shard_group(), pending_tree_diffs, - substate_store.diff() + substate_store.changes() .iter() // Calculate for local shards only and the global shard .filter(|ch| local_committee_info.shard_group().contains_or_global(&ch.shard())), diff --git a/crates/consensus/src/hotstuff/on_ready_to_vote_on_local_block.rs b/crates/consensus/src/hotstuff/on_ready_to_vote_on_local_block.rs index e7ce8d35c6..d8f6b5a8dc 100644 --- a/crates/consensus/src/hotstuff/on_ready_to_vote_on_local_block.rs +++ b/crates/consensus/src/hotstuff/on_ready_to_vote_on_local_block.rs @@ -10,7 +10,13 @@ use tari_engine_types::{ commit_result::{AbortReason, RejectReason}, substate::Substate, }; -use tari_ootle_common_types::{committee::CommitteeInfo, optional::Optional, Epoch, ShardGroup, SubstateAddress}; +use tari_ootle_common_types::{ + committee::CommitteeInfo, + optional::Optional, + Epoch, + ShardGroup, + VersionedSubstateIdRef, +}; use tari_ootle_storage::{ consensus_models::{ Block, @@ -135,7 +141,7 @@ where TConsensusSpec: ConsensusSpec tx, |tx, _prev_locked, block, _justify_qc| self.on_lock_block(tx, block), |tx, mut commit_block| { - let committed = self.on_commit(tx, &block_qc_id, &commit_block, local_committee_info)?; + let committed = self.on_commit(tx, &block_qc_id, &commit_block)?; // NOTE: update the commit QC in the local copy so that foreign proposals can obtain the commit QC // on_commit already sets the persisted commit_qc for the block commit_block.set_commit_qc(block_qc_id); @@ -478,7 +484,7 @@ where TConsensusSpec: ConsensusSpec block.shard_group(), pending, substate_store - .diff() + .changes() .iter() // Calculate for local shards only or the global shard .filter(|ch| block.shard_group().contains_or_global(&ch.shard())), @@ -492,17 +498,19 @@ where TConsensusSpec: ConsensusSpec expected_merkle_root ); let (diff, locks) = substate_store.into_parts(); + let diff = BlockDiff::new(*block.id(), diff); proposed_block_change_set .set_no_vote(NoVoteReason::StateMerkleRootMismatch) // These are set for debugging purposes but aren't actually committed - .set_substate_changes(diff) + .set_block_diff_to_commit(diff.into_filtered(local_committee_info.shard_group())) .set_substate_locks(locks); return Ok(()); } let (diff, locks) = substate_store.into_parts(); + let diff = BlockDiff::new(*block.id(), diff); proposed_block_change_set - .set_substate_changes(diff) + .set_block_diff_to_commit(diff) .set_state_tree_diffs(tree_diffs) .set_substate_locks(locks) .set_quorum_decision(QuorumDecision::Accept); @@ -1485,7 +1493,7 @@ where TConsensusSpec: ConsensusSpec return Ok(Some(NoVoteReason::MintConfidentialOutputUnknown)); }; let substate_id = atom.commitment.into(); - let addr = SubstateAddress::from_substate_id(&substate_id, 0); + let addr = VersionedSubstateIdRef::new(&substate_id, 0); let shard = addr.to_shard(local_committee_info.num_preshards()); let change = SubstateChange::Up { id: substate_id, @@ -1545,9 +1553,8 @@ where TConsensusSpec: ConsensusSpec tx: &mut ::WriteTransaction<'_>, commit_qc_id: &QcId, block: &Block, - local_committee_info: &CommitteeInfo, ) -> Result, HotStuffError> { - let committed_transactions = self.finalize_block(tx, commit_qc_id, block, local_committee_info)?; + let committed_transactions = self.finalize_block(tx, commit_qc_id, block)?; debug!( target: LOG_TARGET, "✅ COMMIT block {}", @@ -1598,7 +1605,6 @@ where TConsensusSpec: ConsensusSpec tx: &mut ::WriteTransaction<'_>, commit_qc_id: &QcId, block: &Block, - local_committee_info: &CommitteeInfo, ) -> Result, HotStuffError> { if block.is_dummy() { block.increment_leader_failure_count( @@ -1607,7 +1613,7 @@ where TConsensusSpec: ConsensusSpec )?; // Nothing to do here for empty dummy blocks. Just mark the block as committed. - block.commit_diff(tx, commit_qc_id, BlockDiff::empty(*block.id()))?; + block.commit_block_without_state_changes(tx, commit_qc_id)?; return Ok(vec![]); } @@ -1632,18 +1638,12 @@ where TConsensusSpec: ConsensusSpec // NOTE: this must happen before we commit the substate diff because the state transitions use this version let pending = block.remove_pending_tree_diff_and_return(tx)?; let mut state_tree = ShardedStateTree::new(tx); - state_tree.commit_diffs(pending)?; + let version_updates = state_tree.commit_diffs(pending)?; let tx = state_tree.into_transaction(); - let diff = block.get_diff(&**tx)?; - info!( - target: LOG_TARGET, - "🌳 COMMIT block {} with {} substate change(s)", block, diff.len() - ); { let _timer = TraceTimer::debug(LOG_TARGET, "commit_block"); - let local_diff = diff.into_filtered(local_committee_info); - block.commit_diff(tx, commit_qc_id, local_diff)?; + block.commit_block(tx, commit_qc_id, &version_updates)?; } let finalized_transactions = { diff --git a/crates/consensus/src/hotstuff/on_receive_local_proposal.rs b/crates/consensus/src/hotstuff/on_receive_local_proposal.rs index ecf5b1b003..078dcf0802 100644 --- a/crates/consensus/src/hotstuff/on_receive_local_proposal.rs +++ b/crates/consensus/src/hotstuff/on_receive_local_proposal.rs @@ -28,7 +28,6 @@ use tari_ootle_storage::{ consensus_models::{ Block, BookkeepingModel, - EpochStateRoot, ForeignProposalRecord, ForeignProposalStatus, NoVoteReason, @@ -480,9 +479,20 @@ impl OnReceiveLocalProposalHandler { info!(target: LOG_TARGET, "⚠️ Behind peers, starting sync ({err})"); diff --git a/crates/consensus/src/hotstuff/substate_store/pending_store.rs b/crates/consensus/src/hotstuff/substate_store/pending_store.rs index 71425c829b..b7017b3604 100644 --- a/crates/consensus/src/hotstuff/substate_store/pending_store.rs +++ b/crates/consensus/src/hotstuff/substate_store/pending_store.rs @@ -1,7 +1,7 @@ // Copyright 2024 The Tari Project // SPDX-License-Identifier: BSD-3-Clause -use std::{borrow::Cow, collections::HashMap, fmt::Display}; +use std::{borrow::Cow, collections::HashMap, fmt::Display, iter}; use indexmap::IndexMap; use log::*; @@ -42,7 +42,7 @@ pub struct PendingSubstateStore<'store, 'tx, TStore: StateStore + 'store + 'tx> /// Map from substate id to the index in the diff list of the latest change head: HashMap, /// Append only list of changes ordered oldest to newest - diff: Vec, + changes: Vec, new_locks: IndexMap>, parent_block: LeafBlock, num_preshards: NumPreshards, @@ -54,7 +54,7 @@ impl<'a, 'tx, TStore: StateStore + 'a> PendingSubstateStore<'a, 'tx, TStore> { store, pending: HashMap::new(), head: HashMap::new(), - diff: Vec::new(), + changes: Vec::new(), new_locks: IndexMap::new(), parent_block, num_preshards, @@ -78,15 +78,15 @@ impl<'a, 'tx, TStore: StateStore + 'a> PendingSubstateStore<'a, 'tx, TStore> { .ok_or_else(|| SubstateStoreError::SubstateNotFound { id: VersionedSubstateId::new(id.clone(), 0), })?; - if let Some(destroyed) = substate.destroyed() { + if substate.is_destroyed() { return Ok(SubstateChange::Down { id: VersionedSubstateId::new(id.clone(), substate.version()), - shard: destroyed.by_shard, + shard: substate.shard(), }); } Ok(SubstateChange::Up { id: id.clone(), - shard: substate.created_by_shard, + shard: substate.shard(), substate: Box::new( substate .into_substate() @@ -131,7 +131,7 @@ impl<'a, 'tx, TStore: StateStore + 'a> PendingSubstateStore<'a, 'tx, TStore> { let Some(change) = self.get_latest_change_from_store(substate_id).optional()? else { debug!(target: LOG_TARGET, "Creating substate in place: {} v0", substate_id); let value = creator(None)?; - let id = SubstateAddress::from_substate_id(substate_id, 0); + let id = VersionedSubstateIdRef::new(substate_id, 0); let up = SubstateChange::Up { shard: id.to_shard(num_preshards), substate: Box::new(Substate::new(0, value)), @@ -254,8 +254,8 @@ impl<'a, 'tx, TStore: StateStore + 'a + 'tx> WriteableSubstateStore for PendingS if id.is_validator_fee_pool() { continue; } - let addr = SubstateAddress::from_substate_id(id, substate.version()); - let shard = addr.to_shard(self.num_preshards); + let vid = VersionedSubstateIdRef::new(id, substate.version()); + let shard = vid.to_shard(self.num_preshards); debug!(target: LOG_TARGET, "🔼️ Up: {} v{} {} value hash: {}", id, substate.version(), shard, substate.to_value_hash()); self.put(SubstateChange::Up { id: id.clone(), @@ -309,7 +309,7 @@ impl<'store, 'tx, TStore: StateStore + 'store + 'tx> PendingSubstateStore<'store if let Some(ch) = self .head .get(id) - .map(|&pos| self.diff.get(pos).expect("diff and head are not in sync")) + .map(|&pos| self.changes.get(pos).expect("diff and head are not in sync")) { return Ok(LatestSubstateVersion { version: ch.versioned_substate_id().version(), @@ -353,13 +353,13 @@ impl<'store, 'tx, TStore: StateStore + 'store + 'tx> PendingSubstateStore<'store fn get_head_change(&self, id: &SubstateId) -> Option<&SubstateChange> { self.head .get(id) - .map(|&pos| self.diff.get(pos).expect("diff and head are not in sync")) + .map(|&pos| self.changes.get(pos).expect("diff and head are not in sync")) } fn get_head_change_mut(&mut self, id: &SubstateId) -> Option<&mut SubstateChange> { self.head .get(id) - .map(|&pos| self.diff.get_mut(pos).expect("diff and head are not in sync")) + .map(|&pos| self.changes.get_mut(pos).expect("diff and head are not in sync")) } pub fn get_latest_change(&self, id: &SubstateId) -> Result { @@ -697,15 +697,15 @@ impl<'store, 'tx, TStore: StateStore + 'store + 'tx> PendingSubstateStore<'store fn get_pending(&self, addr: &SubstateAddress) -> Option<&SubstateChange> { self.pending .get(addr) - .map(|&pos| self.diff.get(pos).expect("pending map and diff are out of sync")) + .map(|&pos| self.changes.get(pos).expect("pending map and diff are out of sync")) } fn insert(&mut self, change: SubstateChange) { - let index = self.diff.len(); + let index = self.changes.len(); self.pending.insert(change.to_substate_address(), index); self.head .insert(change.versioned_substate_id().substate_id().clone(), index); - self.diff.push(change) + self.changes.push(change) } pub fn get_latest_lock_by_id( @@ -853,12 +853,47 @@ impl<'store, 'tx, TStore: StateStore + 'store + 'tx> PendingSubstateStore<'store &self.new_locks } - pub fn diff(&self) -> &Vec { - &self.diff + pub fn changes(&self) -> &Vec { + &self.changes + } + + pub fn generate_shard_state_change_bitmap(&self) -> Vec { + let shard_group = self.parent_block.shard_group(); + let mut bitmap = iter::repeat_n(false, shard_group.len() + 1).collect::>(); + debug!( + target: LOG_TARGET, + "Generating shard state change bitmap (len:{})for shard group {}", + bitmap.len(), + shard_group, + ); + for ch in &self.changes { + let shard = ch.shard(); + if !shard_group.contains_or_global(&shard) { + continue; + } + if shard.is_global() { + bitmap[0] = true; + continue; + } + + let index = shard + .as_u32() + .checked_sub(shard_group.start().as_u32()) + // All values are previously validated + .expect("BUG(to_partial_shard_state_versions): Shard less than shard group start") + as usize + + 1; + assert!( + index < bitmap.len(), + "BUG(to_partial_shard_state_versions): (index: {index}, shard: {shard}, shard group: {shard_group})" + ); + bitmap[index] = true; + } + bitmap } pub fn into_parts(self) -> (Vec, IndexMap>) { - (self.diff, self.new_locks) + (self.changes, self.new_locks) } } diff --git a/crates/consensus/src/hotstuff/substate_store/shard_state_store.rs b/crates/consensus/src/hotstuff/substate_store/shard_state_store.rs index c4f43f7c7e..64d7e7fad4 100644 --- a/crates/consensus/src/hotstuff/substate_store/shard_state_store.rs +++ b/crates/consensus/src/hotstuff/substate_store/shard_state_store.rs @@ -6,7 +6,16 @@ use std::ops::Deref; use log::*; use tari_ootle_common_types::{optional::Optional, shard::Shard}; use tari_ootle_storage::{StateStoreReadTransaction, StateStoreWriteTransaction}; -use tari_state_tree::{JmtStorageError, Node, NodeKey, StaleTreeNode, TreeStoreBatchWriter, TreeStoreReader, Version}; +use tari_state_tree::{ + JmtStorageError, + Node, + NodeKey, + StaleTreeNode, + StateTreePayload, + TreeStoreBatchWriter, + TreeStoreReader, + Version, +}; const LOG_TARGET: &str = "tari::ootle::consensus::sharded_state_tree"; @@ -23,8 +32,8 @@ impl<'a, TTx> ShardScopedTreeStoreReader<'a, TTx> { } } -impl TreeStoreReader for ShardScopedTreeStoreReader<'_, TTx> { - fn get_node(&self, key: &NodeKey) -> Result, tari_state_tree::JmtStorageError> { +impl TreeStoreReader for ShardScopedTreeStoreReader<'_, TTx> { + fn get_node(&self, key: &NodeKey) -> Result, tari_state_tree::JmtStorageError> { self.tx .state_tree_nodes_get(self.shard, key) .optional() @@ -50,7 +59,7 @@ impl<'a, TTx: StateStoreWriteTransaction> ShardScopedTreeStoreWriter<'a, TTx> { Self { shard, tx } } - pub fn set_version(&mut self, version: Version) -> Result<(), tari_state_tree::JmtStorageError> { + pub fn set_state_version(&mut self, version: Version) -> Result<(), tari_state_tree::JmtStorageError> { self.tx .state_tree_shard_versions_set(self.shard, version) .map_err(|e| tari_state_tree::JmtStorageError::UnexpectedError(e.to_string())) @@ -68,7 +77,7 @@ impl<'a, TTx: StateStoreWriteTransaction> ShardScopedTreeStoreWriter<'a, TTx> { pub fn insert_nodes( &mut self, - nodes: Vec<(NodeKey, Node)>, + nodes: Vec<(NodeKey, Node)>, ) -> Result<(), tari_state_tree::JmtStorageError> { self.tx .state_tree_nodes_batch_insert(self.shard, nodes) @@ -80,12 +89,12 @@ impl<'a, TTx: StateStoreWriteTransaction> ShardScopedTreeStoreWriter<'a, TTx> { } } -impl TreeStoreReader for ShardScopedTreeStoreWriter<'_, TTx> +impl TreeStoreReader for ShardScopedTreeStoreWriter<'_, TTx> where TTx: StateStoreWriteTransaction + Deref, TTx::Target: StateStoreReadTransaction, { - fn get_node(&self, key: &NodeKey) -> Result, tari_state_tree::JmtStorageError> { + fn get_node(&self, key: &NodeKey) -> Result, tari_state_tree::JmtStorageError> { self.tx .state_tree_nodes_get(self.shard, key) .optional() @@ -100,8 +109,8 @@ where } } -impl TreeStoreBatchWriter for ShardScopedTreeStoreWriter<'_, TTx> { - fn batch_insert_nodes(&mut self, nodes: Vec<(NodeKey, Node)>) -> Result<(), JmtStorageError> { +impl TreeStoreBatchWriter for ShardScopedTreeStoreWriter<'_, TTx> { + fn batch_insert_nodes(&mut self, nodes: Vec<(NodeKey, Node)>) -> Result<(), JmtStorageError> { self.tx .state_tree_nodes_batch_insert(self.shard, nodes) .map_err(|e| tari_state_tree::JmtStorageError::UnexpectedError(e.to_string())) diff --git a/crates/consensus/src/hotstuff/substate_store/sharded_state_tree.rs b/crates/consensus/src/hotstuff/substate_store/sharded_state_tree.rs index 853f99b889..17d23dbee2 100644 --- a/crates/consensus/src/hotstuff/substate_store/sharded_state_tree.rs +++ b/crates/consensus/src/hotstuff/substate_store/sharded_state_tree.rs @@ -18,6 +18,7 @@ use tari_state_tree::{ StagedTreeStore, StateHashTreeDiff, StateTreeError, + StateTreePayload, SubstateTreeChange, TreeHash, Version, @@ -120,6 +121,15 @@ impl ShardedStateTree<&TTx> { Ok(root_hash) } + pub fn calculate_state_root(&self, shard_group: ShardGroup) -> Result { + let mut shard_state_roots = HashMap::new(); + for shard in shard_group.shard_iter_with_global() { + let root = self.get_state_root_for_shard(shard)?; + shard_state_roots.insert(shard, root); + } + self.get_shard_group_root(shard_group, shard_state_roots) + } + fn get_shard_group_root( &self, shard_group: ShardGroup, @@ -170,7 +180,7 @@ impl ShardedStateTree<&mut TTx> { pub fn commit_diffs( &mut self, diffs: IndexMap>, - ) -> Result<(), StateTreeError> { + ) -> Result, StateTreeError> { debug!( target: LOG_TARGET, "Committing {} pending diff(s) for {} shard(s)", @@ -178,20 +188,22 @@ impl ShardedStateTree<&mut TTx> { diffs.len() ); + let mut state_versions = HashMap::with_capacity(diffs.len()); for (shard, pending_diffs) in diffs { for pending_diff in pending_diffs { + state_versions.insert(shard, pending_diff.version); self.commit_diff(shard, pending_diff.version, pending_diff.diff)?; } } - Ok(()) + Ok(state_versions) } pub fn commit_diff( &mut self, shard: Shard, version: Version, - diff: StateHashTreeDiff, + diff: StateHashTreeDiff, ) -> Result<(), StateTreeError> { let mut store = ShardScopedTreeStoreWriter::new(self.tx, shard); @@ -203,7 +215,7 @@ impl ShardedStateTree<&mut TTx> { ); store.record_stale_tree_nodes(version, diff.stale_tree_nodes)?; store.insert_nodes(diff.new_nodes)?; - store.set_version(version)?; + store.set_state_version(version)?; Ok(()) } } diff --git a/crates/consensus/src/hotstuff/worker.rs b/crates/consensus/src/hotstuff/worker.rs index aca5e499a0..2479f3f64a 100644 --- a/crates/consensus/src/hotstuff/worker.rs +++ b/crates/consensus/src/hotstuff/worker.rs @@ -22,10 +22,8 @@ use tari_ootle_common_types::{optional::Optional, Epoch, NodeHeight, ShardGroup} use tari_ootle_storage::{ consensus_models::{ Block, - BlockDiff, BookkeepingModel, BurntUtxo, - EpochStateRoot, ForeignProposalRecord, NoVoteReason, TransactionPool, @@ -34,7 +32,6 @@ use tari_ootle_storage::{ StateStore, }; use tari_shutdown::ShutdownSignal; -use tari_state_tree::SPARSE_MERKLE_PLACEHOLDER_HASH; use tari_template_lib_types::crypto::RistrettoPublicKeyBytes; use tari_transaction::{Transaction, TransactionId}; use tokio::sync::{broadcast, mpsc}; @@ -66,6 +63,7 @@ use crate::{ pacemaker::PaceMaker, pacemaker_handle::PaceMakerHandle, state_tree_gc::StateTreeGc, + substate_store::ShardedStateTree, transaction_manager::ConsensusTransactionManager, vote_collector::{ProposalVoteCollector, TimeoutVoteCollector}, }, @@ -1003,7 +1001,7 @@ impl HotstuffWorker { }; if remote_epoch > local_epoch_state.epoch() { - // We are in a future epoch, so we cannot justify this block + // Valid remote certificate is in a future epoch, so we are behind warn!( target: LOG_TARGET, "❌ Justify block {remote_epoch}/{remote_height} is in a future epoch > current epoch {}. State sync required.", @@ -1054,17 +1052,11 @@ impl HotstuffWorker { zero_block.justify().save(tx)?; zero_block.insert(tx)?; zero_block.add_justify_qc(tx, &QcId::zero())?; - zero_block.commit_diff( - tx, - &zero_block.justify().calculate_id(), - BlockDiff::empty(*zero_block.id()), - )?; + zero_block.commit_block_without_state_changes(tx, &zero_block.justify().calculate_id())?; } - let checkpoint = EpochStateRoot::get(&**tx).optional()?; - let state_merkle_root = checkpoint - .map(|cp| cp.state_root) - .unwrap_or_else(|| SPARSE_MERKLE_PLACEHOLDER_HASH); + let state_merkle_root = ShardedStateTree::new(&**tx).calculate_state_root(shard_group)?; + let mut genesis = Block::genesis( self.config.network, epoch, @@ -1084,7 +1076,7 @@ impl HotstuffWorker { genesis.as_last_executed().set(tx)?; genesis.as_last_voted().set(tx)?; genesis.justify().as_high_pc().set(tx)?; - genesis.commit_diff(tx, &genesis.justify().calculate_id(), BlockDiff::empty(*genesis.id()))?; + genesis.commit_block_without_state_changes(tx, &genesis.justify().calculate_id())?; } Ok(()) diff --git a/crates/consensus_tests/fixtures/block.json b/crates/consensus_tests/fixtures/block.json index 668ac8b512..405db0ccac 100644 --- a/crates/consensus_tests/fixtures/block.json +++ b/crates/consensus_tests/fixtures/block.json @@ -7,7 +7,7 @@ "height": 66, "epoch": 5, "shard_group": { - "start": 0, + "start": 1, "end_inclusive": 255 }, "proposed_by": "5e13c16840aa8d2e7e68390d0eb1b45c86bc363db0419f7ec5daa534e63bdb35", diff --git a/crates/consensus_tests/fixtures/block_with_dummies.json b/crates/consensus_tests/fixtures/block_with_dummies.json index d049bb6374..d0d2c19d65 100644 --- a/crates/consensus_tests/fixtures/block_with_dummies.json +++ b/crates/consensus_tests/fixtures/block_with_dummies.json @@ -7,7 +7,7 @@ "height": 18, "epoch": 5, "shard_group": { - "start": 0, + "start": 1, "end_inclusive": 127 }, "proposed_by": "584b42ac801387ab50e4af50aaea63ff9cf0b4ae8fe199157e811b0eae67cf51", diff --git a/crates/consensus_tests/src/consensus.rs b/crates/consensus_tests/src/consensus.rs index 29100ccece..3f896faf92 100644 --- a/crates/consensus_tests/src/consensus.rs +++ b/crates/consensus_tests/src/consensus.rs @@ -30,9 +30,10 @@ use tari_ootle_common_types::{ shard::Shard, Epoch, NodeHeight, - SubstateAddress, SubstateLockType, SubstateRequirement, + ToSubstateAddress, + VersionedSubstateId, }; use tari_ootle_storage::{ consensus_models::{Block, Command, SubstateRecord, TransactionRecord}, @@ -49,7 +50,6 @@ use crate::support::{ Test, TestAddress, TestVnDestination, - TEST_NUM_PRESHARDS, }; // Although these tests will pass with a single thread, we enable multi-threaded mode so that any unhandled race @@ -1400,34 +1400,20 @@ async fn multishard_publish_template() { test.assert_all_validators_committed(tx.id()); // Assert all have the template - let address = SubstateAddress::from_substate_id(&template_id.into(), 0); - // Figure out which VN is responsible for the template substate - let shard = address.to_shard(TEST_NUM_PRESHARDS); - let vn_addr = test - .num_preshards() - .all_shard_groups_iter(test.num_committees()) - .enumerate() - .find_map(|(i, sg)| { - if sg.contains(&shard) { - Some(TestAddress::new(((i + 1) * 2).to_string())) - } else { - None - } - }) - .unwrap(); - - let template_substate = test - .get_validator(&vn_addr) - .state_store - .with_read_tx(|tx| SubstateRecord::get(tx, &address)) - .unwrap_or_else(|e| panic!("Failed to get template substate from {vn_addr}: {e}")); - let binary_hash = template_substate - .substate_value - .unwrap() - .into_template() - .expect("Expected template substate") - .binary_hash; - assert_eq!(binary_hash, hash_template_code(&wasm), "Template binary does not match"); + for (addr, vn) in test.validators() { + let substate_addr = VersionedSubstateId::new(template_id, 0).to_substate_address(); + let template_substate = vn + .state_store + .with_read_tx(|tx| SubstateRecord::get(tx, &substate_addr)) + .unwrap_or_else(|e| panic!("Failed to get template substate from {addr}: {e}")); + let binary_hash = template_substate + .substate_value + .unwrap() + .into_template() + .expect("Expected template substate") + .binary_hash; + assert_eq!(binary_hash, hash_template_code(&wasm), "Template binary does not match"); + } test.assert_clean_shutdown().await; } diff --git a/crates/consensus_tests/src/dummy_blocks.rs b/crates/consensus_tests/src/dummy_blocks.rs index d029b83d0d..d728c13123 100644 --- a/crates/consensus_tests/src/dummy_blocks.rs +++ b/crates/consensus_tests/src/dummy_blocks.rs @@ -25,17 +25,18 @@ use crate::support::{load_json_fixture, RoundRobinLeaderStrategy}; #[test] fn dummy_blocks() { + let shard_group = ShardGroup::new(1, 127); let genesis = Block::genesis( Network::LocalNet, Epoch(1), FixedHash::zero(), - ShardGroup::new(0, 127), + shard_group, FixedHash::zero(), None, ); let committee = (0u8..2) - .map(|i| create_key_pair_from_seed(i).1) - .map(|pk| CommitteeMember { + .map(create_key_pair_from_seed) + .map(|(_, pk)| CommitteeMember { address: PeerAddress::derive_from_public_key(&pk), public_key: pk.to_byte_type(), vote_power: VotePower::of(1), @@ -47,7 +48,7 @@ fn dummy_blocks() { NodeHeight(30), Network::LocalNet, Epoch(1), - ShardGroup::new(0, 127), + shard_group, *genesis.id(), genesis.justify(), genesis.id(), @@ -62,7 +63,7 @@ fn dummy_blocks() { NodeHeight(30), Network::LocalNet, Epoch(1), - ShardGroup::new(0, 127), + shard_group, *genesis.id(), genesis.justify(), FixedHash::zero(), diff --git a/crates/consensus_tests/src/state_tree.rs b/crates/consensus_tests/src/state_tree.rs index 2b0547f2fa..6313985000 100644 --- a/crates/consensus_tests/src/state_tree.rs +++ b/crates/consensus_tests/src/state_tree.rs @@ -5,9 +5,13 @@ use std::time::Duration; use tari_consensus::hotstuff::HotStuffError; use tari_consensus_types::Decision; -use tari_ootle_common_types::{Epoch, NodeHeight}; -use tari_ootle_storage::{consensus_models::StateTransitionId, StateStore, StateStoreReadTransaction}; -use tari_state_tree::{key_mapper::SpreadPrefixKeyMapper, memory_store::MemoryTreeStore}; +use tari_ootle_common_types::{optional::Optional, Epoch, NodeHeight}; +use tari_ootle_storage::{StateStore, StateStoreReadTransaction}; +use tari_state_tree::{ + key_mapper::SpreadPrefixKeyMapper, + memory_store::MemoryTreeStore, + SPARSE_MERKLE_PLACEHOLDER_HASH, +}; use crate::support::{logging::setup_logger, Test, TestAddress, TEST_NUM_PRESHARDS}; @@ -15,6 +19,7 @@ use crate::support::{logging::setup_logger, Test, TestAddress, TEST_NUM_PRESHARD async fn check_state_transitions() { setup_logger(); let mut test = Test::builder() + .with_rocks_path("/tmp/test{}") .modify_consensus_constants(|config| { config.pacemaker_block_time = Duration::from_millis(500); }) @@ -24,6 +29,7 @@ async fn check_state_transitions() { let _ignore = test.send_transaction_to_all(Decision::Commit, 100, 1, 10).await; let _ignore = test.send_transaction_to_all(Decision::Commit, 200, 1, 1).await; let _ignore = test.send_transaction_to_all(Decision::Commit, 1, 1, 1).await; + let _ignore = test.send_transaction_to_all(Decision::Commit, 100, 1, 10).await; test.start_epoch(Epoch(1)).await; loop { @@ -49,26 +55,57 @@ async fn check_state_transitions() { } } + test.stop(); + test.get_validator(&TestAddress::new("1")) .state_store .with_read_tx(|tx| { let checkpoint = tx.epoch_checkpoint_get(Epoch(1)).unwrap(); for shard in TEST_NUM_PRESHARDS.all_shards_iter() { - let id = StateTransitionId::new(Epoch(0), shard, 0); - let transitions = tx.state_transitions_get_n_after(1000, id, Epoch(2)).unwrap(); + let mut all_transitions = vec![]; + let mut next_state_version = 1; + while let Some(transitions) = tx + .state_transitions_get_after(shard, next_state_version, false) + .optional() + .unwrap() + { + if transitions.epoch > checkpoint.epoch() { + break; + } + + next_state_version = transitions.state_version + 1; + all_transitions.push(transitions); + } + log::info!( + "Shard {}: Found {} transitions until state version {}", + shard, + all_transitions.len(), + next_state_version + ); let shard_root = checkpoint.get_shard_root(shard); // No state changes - if shard_root.iter().all(|x| *x == 0) { - assert_eq!(transitions.len(), 0, "Shard {} should have no state transitions", shard); + if shard_root == SPARSE_MERKLE_PLACEHOLDER_HASH { + assert!( + all_transitions.is_empty(), + "Shard {} should have no state transitions", + shard + ); } else { - assert!(!transitions.is_empty(), "Shard {} should have state transitions", shard); + assert!( + !all_transitions.is_empty(), + "Shard {} should have state transitions", + shard + ); } let mut store = MemoryTreeStore::new(); let mut tree = tari_state_tree::StateTree::<_, SpreadPrefixKeyMapper>::new(&mut store); - let values = transitions.iter().map(|transition| transition.to_tree_change()); + let values = all_transitions + .iter() + .flat_map(|t| &t.updates) + .map(|transition| transition.to_tree_change()); let root = tree.put_substate_changes(None, 1, values).unwrap(); assert_eq!(root, shard_root, "Shard {} root hash mismatch", shard); } diff --git a/crates/consensus_tests/src/substate_store.rs b/crates/consensus_tests/src/substate_store.rs index 93661d2fdc..a1c69d835b 100644 --- a/crates/consensus_tests/src/substate_store.rs +++ b/crates/consensus_tests/src/substate_store.rs @@ -5,7 +5,7 @@ use tari_consensus::{ hotstuff::substate_store::{LockFailedError, PendingSubstateStore, SubstateStoreError}, traits::{CertificateStore, ReadableSubstateStore, WriteableSubstateStore}, }; -use tari_consensus_types::{BlockId, LeafBlock, QcId}; +use tari_consensus_types::{BlockId, LeafBlock}; use tari_engine_types::{ component::{ComponentBody, ComponentHeader}, substate::{Substate, SubstateId, SubstateValue}, @@ -22,7 +22,14 @@ use tari_ootle_common_types::{ VersionedSubstateId, }; use tari_ootle_storage::{ - consensus_models::{Block, RequireLockIntentRef, SubstateChange, SubstateRecord}, + consensus_models::{ + Block, + RequireLockIntentRef, + SubstateChange, + SubstateRecord, + SubstateTransition, + SubstateUpdateBatch, + }, StateStore, }; use tari_state_store_rocksdb::{DatabaseOptions, RocksDbStateStore}; @@ -207,22 +214,15 @@ fn it_allows_requesting_the_same_lock_within_one_transaction() { fn add_substate(store: &TestStore, seed: u8, version: u32) -> VersionedSubstateId { let id = new_substate_id(seed); let value = new_substate_value(seed); + let mut batch = SubstateUpdateBatch::new(Epoch::zero()); + batch.add_transition(Shard::first(), 0, SubstateTransition::Up { + id: id.clone(), + version, + substate_or_hash: value.into(), + }); store - .with_write_tx(|tx| { - SubstateRecord { - substate_id: id.clone(), - version, - substate_value: Some(value), - state_hash: [seed; 32].into(), - created_justify: QcId::zero(), - created_block: BlockId::zero(), - created_by_shard: Shard::first(), - created_at_epoch: 0.into(), - destroyed: None, - } - .create(tx) - }) + .with_write_tx(|tx| SubstateRecord::commit_batch(tx, batch)) .unwrap(); VersionedSubstateId::new(id, version) diff --git a/crates/consensus_tests/src/support/harness.rs b/crates/consensus_tests/src/support/harness.rs index f7a392a2f1..66d5972910 100644 --- a/crates/consensus_tests/src/support/harness.rs +++ b/crates/consensus_tests/src/support/harness.rs @@ -15,7 +15,7 @@ use tari_consensus::{ consensus_constants::ConsensusConstants, hotstuff::{HotstuffConfig, HotstuffEvent}, }; -use tari_consensus_types::{BlockId, Decision, QcId}; +use tari_consensus_types::{BlockId, Decision}; use tari_crypto::ristretto::RistrettoPublicKey; use tari_engine_types::{substate::SubstateId, ToByteType}; use tari_epoch_manager::EpochManagerReader; @@ -34,7 +34,7 @@ use tari_ootle_common_types::{ VotePower, }; use tari_ootle_storage::{ - consensus_models::{SubstateRecord, TransactionExecution, TransactionRecord}, + consensus_models::{SubstateCreated, SubstateRecord, SubstateUpdateBatch, TransactionExecution, TransactionRecord}, StateStore, StateStoreReadTransaction, StorageError, @@ -179,30 +179,29 @@ impl Test { .iter() .map(|id| { let value = make_test_component(id.substate_id().as_component_address().unwrap().entity_id()); - SubstateRecord::new( - id.substate_id().clone(), - id.version(), - value, - Shard::first(), - Epoch(0), - BlockId::zero(), - QcId::zero(), - ) + SubstateRecord::new(id.substate_id().clone(), id.version(), value, SubstateCreated { + at_epoch: Epoch::zero(), + in_shard: Shard::first(), + at_state_version: 0, + }) }) .collect::>(); self.validators.values().filter(|vn| dest.is_for_vn(vn)).for_each(|v| { + let mut batch = SubstateUpdateBatch::new(Epoch::zero()); + for substate in &substates { + let shard = substate.to_versioned_substate_id().to_shard(TEST_NUM_PRESHARDS); + if v.shard_group.contains(&shard) { + batch.add_transition( + shard, + substate.created().at_state_version, + substate.clone().into_transition(), + ); + } + } + v.state_store - .with_write_tx(|tx| { - for substate in &substates { - if v.shard_group - .contains(&substate.to_substate_address().to_shard(TEST_NUM_PRESHARDS)) - { - substate.create(tx).unwrap(); - } - } - Ok::<_, StorageError>(()) - }) + .with_write_tx(|tx| SubstateRecord::commit_batch(tx, batch)) .unwrap(); }); @@ -227,6 +226,7 @@ impl Test { TEST_NUM_PRESHARDS } + #[allow(dead_code)] pub fn num_committees(&self) -> u32 { self.num_committees } diff --git a/crates/consensus_tests/src/support/helpers.rs b/crates/consensus_tests/src/support/helpers.rs index eed3f72e00..1cad308623 100644 --- a/crates/consensus_tests/src/support/helpers.rs +++ b/crates/consensus_tests/src/support/helpers.rs @@ -32,6 +32,7 @@ pub(crate) fn random_substate_in_shard_group(shard_group: ShardGroup, num_shards SubstateId::Component(ComponentAddress::new(ObjectKey::new(entity_id, component_key))) } +// TODO: this biases the start of the shard group fn random_substate_address_range(range: RangeInclusive) -> SubstateAddress { let start = range.start(); let mut bytes = [0u8; 16]; diff --git a/crates/p2p/proto/consensus.proto b/crates/p2p/proto/consensus.proto index 39af02cfb2..fb17aec039 100644 --- a/crates/p2p/proto/consensus.proto +++ b/crates/p2p/proto/consensus.proto @@ -91,8 +91,8 @@ message BlockHeader { uint64 total_leader_fee = 8; tari.ootle.common.Signature signature = 10; uint64 timestamp = 11; - bytes epoch_hash = 13; - ExtraData extra_data = 14; + bytes epoch_hash = 12; + ExtraData extra_data = 13; } message Block { @@ -102,6 +102,10 @@ message Block { TimeoutCertificate timeout_certificate = 4; // optional } +message ShardStateVersions { + repeated uint64 versions = 1; +} + message ExtraData { bytes encoded_extra_data = 1; } @@ -237,19 +241,21 @@ message Substate { uint32 version = 2; bytes substate = 3; - uint64 created_epoch = 4; - uint32 created_by_shard = 5; - bytes created_block = 6; - bytes created_justify = 7; + // Required + SubstateCreatedMetadata created = 4; + // Optional + SubstateDestroyedMetadata destroyed = 5; +} - SubstateDestroyed destroyed = 8; +message SubstateCreatedMetadata { + tari.ootle.common.Epoch at_epoch = 1; + uint32 in_shard = 2; + uint64 at_state_version = 3; } -message SubstateDestroyed { - tari.ootle.common.Epoch epoch = 1; - uint32 shard = 2; - uint64 block_height = 3; - bytes justify = 4; +message SubstateDestroyedMetadata { + tari.ootle.common.Epoch at_epoch = 1; + uint64 at_state_version = 2; } message SyncRequest { diff --git a/crates/p2p/proto/rpc.proto b/crates/p2p/proto/rpc.proto index c96fecebc9..e4d1f0fd65 100644 --- a/crates/p2p/proto/rpc.proto +++ b/crates/p2p/proto/rpc.proto @@ -99,7 +99,9 @@ message GetSubstateResponse { // Encoded Substate bytes substate = 3; SubstateStatus status = 4; - repeated tari.ootle.consensus.QuorumCertificate quorum_certificates = 5; + uint64 created_at_state_version = 5; + // Optional (i.e. 0 if not destroyed) + uint64 destroyed_at_state_version = 6; } enum SubstateStatus { @@ -202,32 +204,25 @@ message GetCheckpointResponse { message EpochCheckpoint { bytes proof = 1; - map shard_roots = 2; + map shard_tree_summary = 2; } -message SyncStateRequest { - uint64 start_epoch = 1; - uint32 start_shard = 2; - uint64 start_seq = 3; - // The shard in the current shard-epoch that is requested. - // This will limit the state transitions returned to those that fall within this shard-epoch. - uint64 current_epoch = 4; +message TreeRootSummary { + bytes root_hash = 1; + uint64 state_version = 2; } -message SyncStateResponse { - repeated StateTransition transitions = 1; -} - -message StateTransition { - StateTransitionId id = 1; - SubstateUpdate update = 2; - uint64 state_version = 3; +message SyncStateRequest { + uint64 start_state_version = 1; + uint32 shard = 2; + uint64 until_epoch = 3; } -message StateTransitionId { - uint64 epoch = 1; - uint32 shard = 2; - uint64 seq = 3; +message SyncStateResponse { + uint64 state_version = 1; + repeated SubstateUpdate updates = 2; + bool has_more = 3; + tari.ootle.common.Epoch Epoch = 4; } enum TemplateType { diff --git a/crates/p2p/src/conversions/consensus.rs b/crates/p2p/src/conversions/consensus.rs index 075d39d0ba..8f5d645599 100644 --- a/crates/p2p/src/conversions/consensus.rs +++ b/crates/p2p/src/conversions/consensus.rs @@ -52,7 +52,15 @@ use tari_engine_types::{ commit_result::AbortReason, substate::{SubstateId, SubstateValue}, }; -use tari_ootle_common_types::{shard::Shard, Epoch, ExtraData, NodeHeight, ShardGroup, ValidatorMetadata}; +use tari_ootle_common_types::{ + shard::Shard, + Epoch, + ExtraData, + NodeHeight, + ShardGroup, + ShardStateVersions, + ValidatorMetadata, +}; use tari_ootle_storage::{ consensus_models, consensus_models::{ @@ -63,6 +71,7 @@ use tari_ootle_storage::{ ForeignProposalAtom, LeaderFee, MintConfidentialOutputAtom, + SubstateCreated, SubstateDestroyed, SubstateRecord, TransactionAtom, @@ -968,12 +977,11 @@ impl TryFrom for SubstateRecord { // TODO: Should we add this to the proto? state_hash: Default::default(), - created_at_epoch: Epoch(value.created_epoch), - created_justify: value.created_justify.as_slice().try_into()?, - created_block: value.created_block.try_into()?, - + created: value + .created + .ok_or_else(|| anyhow!("Substate created metadata not provided"))? + .try_into()?, destroyed: value.destroyed.map(TryInto::try_into).transpose()?, - created_by_shard: Shard::from(value.created_by_shard), }) } } @@ -985,40 +993,58 @@ impl From for proto::consensus::Substate { version: value.version, substate: value.substate_value.as_ref().map(|s| s.to_bytes()).unwrap_or_default(), - created_justify: value.created_justify.as_bytes().to_vec(), - created_block: value.created_block.as_bytes().to_vec(), - created_epoch: value.created_at_epoch.as_u64(), - created_by_shard: value.created_by_shard.as_u32(), - + created: Some(value.created().into()), destroyed: value.destroyed.map(Into::into), } } } -// -------------------------------- SubstateDestroyed -------------------------------- // -impl TryFrom for SubstateDestroyed { +// -------------------------------- SubstateCreatedMetadata -------------------------------- // +impl TryFrom for SubstateCreated { type Error = anyhow::Error; - fn try_from(value: proto::consensus::SubstateDestroyed) -> Result { + fn try_from(value: proto::consensus::SubstateCreatedMetadata) -> Result { Ok(Self { - justify: value.justify.as_slice().try_into()?, - by_block: NodeHeight(value.block_height), at_epoch: value - .epoch + .at_epoch .map(Into::into) .ok_or_else(|| anyhow!("Epoch not provided"))?, - by_shard: Shard::from(value.shard), + in_shard: Shard::from(value.in_shard), + at_state_version: value.at_state_version, }) } } -impl From for proto::consensus::SubstateDestroyed { +impl From<&SubstateCreated> for proto::consensus::SubstateCreatedMetadata { + fn from(value: &SubstateCreated) -> Self { + Self { + at_epoch: Some(value.at_epoch.into()), + in_shard: value.in_shard.as_u32(), + at_state_version: value.at_state_version, + } + } +} + +// -------------------------------- SubstateDestroyedMetadata -------------------------------- // +impl TryFrom for SubstateDestroyed { + type Error = anyhow::Error; + + fn try_from(value: proto::consensus::SubstateDestroyedMetadata) -> Result { + Ok(Self { + at_epoch: value + .at_epoch + .map(Into::into) + .ok_or_else(|| anyhow!("Epoch not provided"))?, + at_state_version: value.at_state_version, + }) + } +} + +impl From for proto::consensus::SubstateDestroyedMetadata { fn from(value: SubstateDestroyed) -> Self { Self { - justify: value.justify.as_bytes().to_vec(), - block_height: value.by_block.as_u64(), - epoch: Some(value.at_epoch.into()), - shard: value.by_shard.as_u32(), + at_epoch: Some(value.at_epoch.into()), + at_state_version: value.at_state_version, } } } @@ -1044,3 +1070,20 @@ impl TryFrom for SyncRequestMessage { }) } } + +// -------------------------------- ShardStateVersions -------------------------------- // +impl From<&ShardStateVersions> for proto::consensus::ShardStateVersions { + fn from(value: &ShardStateVersions) -> Self { + Self { + versions: value.as_slice().to_vec(), + } + } +} + +impl TryFrom for ShardStateVersions { + type Error = anyhow::Error; + + fn try_from(value: proto::consensus::ShardStateVersions) -> Result { + ShardStateVersions::from_vec(value.versions).map_err(|e| anyhow!("Failed to convert ShardStateVersions: {}", e)) + } +} diff --git a/crates/p2p/src/conversions/rpc.rs b/crates/p2p/src/conversions/rpc.rs index b2f2098819..8aca1c3526 100644 --- a/crates/p2p/src/conversions/rpc.rs +++ b/crates/p2p/src/conversions/rpc.rs @@ -7,16 +7,15 @@ use anyhow::{anyhow, Context}; use tari_common_types::types::FixedHash; use tari_engine_types::substate::{SubstateId, SubstateValue}; use tari_jellyfish::TreeHash; -use tari_ootle_common_types::{shard::Shard, Epoch}; +use tari_ootle_common_types::shard::Shard; use tari_ootle_storage::consensus_models::{ EpochCheckpoint, - StateTransition, - StateTransitionId, SubstateCreatedProof, SubstateData, SubstateDestroyedProof, - SubstateUpdate, + SubstateUpdateProof, SubstateValueOrHash, + TreeRootSummary, }; use crate::{ @@ -34,11 +33,6 @@ impl TryFrom for SubstateCreatedProof { .map(TryInto::try_into) .transpose()? .ok_or_else(|| anyhow!("substate not provided"))?, - // created_qc: value - // .created_justify - // .map(TryInto::try_into) - // .transpose()? - // .ok_or_else(|| anyhow!("created_justify not provided"))?, }) } } @@ -59,11 +53,6 @@ impl TryFrom for SubstateDestroyedProof { Ok(Self { substate_id: SubstateId::from_bytes(&value.substate_id)?, version: value.version, - // justify: value - // .destroyed_justify - // .map(TryInto::try_into) - // .transpose()? - // .ok_or_else(|| anyhow!("destroyed_justify not provided"))?, }) } } @@ -78,7 +67,7 @@ impl From for proto::rpc::SubstateDestroyedProof { } } -impl TryFrom for SubstateUpdate { +impl TryFrom for SubstateUpdateProof { type Error = anyhow::Error; fn try_from(value: proto::rpc::SubstateUpdate) -> Result { @@ -90,11 +79,11 @@ impl TryFrom for SubstateUpdate { } } -impl From for proto::rpc::SubstateUpdate { - fn from(value: SubstateUpdate) -> Self { +impl From for proto::rpc::SubstateUpdate { + fn from(value: SubstateUpdateProof) -> Self { let update = match value { - SubstateUpdate::Create(proof) => proto::rpc::substate_update::Update::Create(proof.into()), - SubstateUpdate::Destroy(proof) => proto::rpc::substate_update::Update::Destroy(proof.into()), + SubstateUpdateProof::Create(proof) => proto::rpc::substate_update::Update::Create(proof.into()), + SubstateUpdateProof::Destroy(proof) => proto::rpc::substate_update::Update::Destroy(proof.into()), }; Self { update: Some(update) } @@ -152,59 +141,6 @@ impl TryFrom for SubstateValueOr } } -//---------------------------------- StateTransition --------------------------------------------// - -impl TryFrom for StateTransition { - type Error = anyhow::Error; - - fn try_from(value: proto::rpc::StateTransition) -> Result { - let id = value - .id - .map(StateTransitionId::try_from) - .transpose()? - .ok_or_else(|| anyhow::anyhow!("StateTransitionId is missing"))?; - let update = value - .update - .ok_or_else(|| anyhow::anyhow!("Missing state transition update"))?; - let update = SubstateUpdate::try_from(update)?; - Ok(Self { - id, - state_version: value.state_version, - update, - }) - } -} - -impl From for proto::rpc::StateTransition { - fn from(value: StateTransition) -> Self { - Self { - id: Some(value.id.into()), - update: Some(value.update.into()), - state_version: value.state_version, - } - } -} - -//---------------------------------- StateTransitionId --------------------------------------------// - -impl TryFrom for StateTransitionId { - type Error = anyhow::Error; - - fn try_from(value: proto::rpc::StateTransitionId) -> Result { - Ok(Self::new(Epoch(value.epoch), Shard::from(value.shard), value.seq)) - } -} - -impl From for proto::rpc::StateTransitionId { - fn from(value: StateTransitionId) -> Self { - Self { - epoch: value.epoch().as_u64(), - shard: value.shard().as_u32(), - seq: value.seq(), - } - } -} - //---------------------------------- EpochCheckpoint --------------------------------------------// impl TryFrom for EpochCheckpoint { @@ -212,17 +148,17 @@ impl TryFrom for EpochCheckpoint { fn try_from(value: proto::rpc::EpochCheckpoint) -> Result { // Defensive check to mitigate DoS attacks - if value.shard_roots.len() > 100_000 { - return Err(anyhow!("too many shard roots (num={})", value.shard_roots.len())); + if value.shard_tree_summary.len() > 100_000 { + return Err(anyhow!("too many shard roots (num={})", value.shard_tree_summary.len())); } - let shard_roots = value - .shard_roots + let shard_tree_summary = value + .shard_tree_summary .into_iter() - .map(|(k, v)| TreeHash::try_from_bytes(&v).map(|h| (Shard::from(k), h))) + .map(|(k, v)| v.try_into().map(|s| (Shard::from(k), s))) .collect::>()?; - Ok(Self::new(decode_from_slice(&value.proof)?, shard_roots)) + Ok(Self::new(decode_from_slice(&value.proof)?, shard_tree_summary)) } } @@ -230,11 +166,32 @@ impl From for proto::rpc::EpochCheckpoint { fn from(value: EpochCheckpoint) -> Self { Self { proof: encode_to_vec(value.proof()).unwrap(), - shard_roots: value - .shard_roots() + shard_tree_summary: value + .shard_tree_summary() .iter() - .map(|(k, v)| (k.as_u32(), v.to_vec())) + .map(|(k, v)| (k.as_u32(), v.into())) .collect(), } } } + +// -------------------------------- TreeRootSummary -------------------------------- // +impl TryFrom for TreeRootSummary { + type Error = anyhow::Error; + + fn try_from(value: proto::rpc::TreeRootSummary) -> Result { + Ok(Self { + root_hash: TreeHash::try_from_bytes(&value.root_hash).context("TreeRootSummary::root_hash")?, + state_version: value.state_version, + }) + } +} + +impl From<&TreeRootSummary> for proto::rpc::TreeRootSummary { + fn from(value: &TreeRootSummary) -> Self { + Self { + root_hash: value.root_hash.as_slice().to_vec(), + state_version: value.state_version, + } + } +} diff --git a/crates/rpc_state_sync/src/state_sync.rs b/crates/rpc_state_sync/src/state_sync.rs index db2d164715..bace074f18 100644 --- a/crates/rpc_state_sync/src/state_sync.rs +++ b/crates/rpc_state_sync/src/state_sync.rs @@ -1,10 +1,7 @@ // Copyright 2023 The Tari Project // SPDX-License-Identifier: BSD-3-Clause -use std::{ - collections::{BTreeMap, HashMap}, - time::Instant, -}; +use std::{collections::HashMap, time::Instant}; use anyhow::anyhow; use futures::StreamExt; @@ -14,7 +11,7 @@ use tari_consensus::{ hotstuff::substate_store::{ShardScopedTreeStoreReader, ShardScopedTreeStoreWriter}, traits::{ConsensusSpec, SyncManager, SyncStatus}, }; -use tari_consensus_types::{BlockId, LeafBlock, QcId}; +use tari_consensus_types::LeafBlock; use tari_epoch_manager::EpochManagerReader; use tari_ootle_common_types::{ committee::Committee, @@ -32,13 +29,12 @@ use tari_ootle_storage::{ consensus_models::{ BookkeepingModel, EpochCheckpoint, - EpochStateRoot, - StateTransition, - StateTransitionId, SubstateCreatedProof, - SubstateDestroyedProof, SubstateRecord, - SubstateUpdate, + SubstateTransition, + SubstateTransitionData, + SubstateUpdateBatch, + SubstateUpdateProof, }, StateStore, StateStoreReadTransaction, @@ -46,14 +42,7 @@ use tari_ootle_storage::{ StorageError, }; use tari_rpc_framework::RpcError; -use tari_state_tree::{ - compute_merkle_root_for_hashes, - SpreadPrefixStateTree, - SubstateTreeChange, - TreeHash, - Version, - SPARSE_MERKLE_PLACEHOLDER_HASH, -}; +use tari_state_tree::{SpreadPrefixStateTree, SubstateTreeChange, TreeHash, Version, SPARSE_MERKLE_PLACEHOLDER_HASH}; use tari_template_manager::interface::{TemplateChange, TemplateManagerHandle}; use tari_validator_node_rpc::{ client::{TariValidatorNodeRpcClientFactory, ValidatorNodeClientFactory}, @@ -125,6 +114,7 @@ where TConsensusSpec: ConsensusSpec Ok(checkpoint) => { info!(target: LOG_TARGET, "🛜 Checkpoint: {checkpoint}"); self.validate_checkpoint(&checkpoint, prev_committee, prev_epoch)?; + self.state_store.with_write_tx(|tx| checkpoint.save(tx))?; self.valid_checkpoints.insert(for_shard_group, checkpoint.clone()); Ok(Some(checkpoint)) }, @@ -143,268 +133,224 @@ where TConsensusSpec: ConsensusSpec client: &mut ValidatorNodeRpcClient, shard: Shard, checkpoint: &EpochCheckpoint, - template_changes_mut: &mut Vec, - ) -> Result, RpcStateSyncError> { - let checkpoint_state_root = checkpoint.get_shard_root(shard); - if checkpoint_state_root == SPARSE_MERKLE_PLACEHOLDER_HASH { - info!(target: LOG_TARGET, "Checkpoint state root indicates no state changes. Nothing to sync for {shard}"); - return Ok(None); - } - - let checkpoint_block_id = BlockId::new(checkpoint.header().calculate_block_id()); - - let current_epoch = self.epoch_manager.current_epoch().await?; - - let last_state_transition_id = self - .state_store - .with_read_tx(|tx| StateTransition::get_last_id(tx, shard)) - .optional()? - .unwrap_or_else(|| StateTransitionId::initial(shard)); + mut maybe_persisted_state_version: Option, + ) -> Result<(Option, Vec), RpcStateSyncError> { + let mut template_changes = vec![]; + let checkpoint_shard_root = checkpoint.get_shard_root(shard); + let checkpoint_state_version = checkpoint.get_shard_state_version(shard); - let mut maybe_persisted_state_version = self + let initial_local_state_root = self .state_store - .with_read_tx(|tx| tx.state_tree_versions_get_latest(shard))?; - - if current_epoch == last_state_transition_id.epoch() { - info!(target: LOG_TARGET, "🛜Already up to date. No need to sync."); - return Ok(maybe_persisted_state_version); + .with_read_tx(|tx| self.calculate_state_root_for_shard(tx, shard, maybe_persisted_state_version))?; + if checkpoint_shard_root == initial_local_state_root { + info!(target: LOG_TARGET, "Checkpoint state root indicates no further state changes. Nothing to sync for {shard}"); + return Ok((None, vec![])); } + // We start at 1 because bootstrapped state is at 0 + let start_state_version = maybe_persisted_state_version.unwrap_or(1); info!( target: LOG_TARGET, - "🛜Syncing from v{} to state transition {last_state_transition_id}", - maybe_persisted_state_version.unwrap_or(0) + "🛜Syncing from v{start_state_version}", ); self.stats.total_requests += 1; let mut state_stream = client .sync_state(SyncStateRequest { - start_epoch: last_state_transition_id.epoch().as_u64(), - start_shard: last_state_transition_id.shard().as_u32(), - start_seq: last_state_transition_id.seq(), - current_epoch: current_epoch.as_u64(), + start_state_version, + shard: shard.as_u32(), + until_epoch: checkpoint.epoch().as_u64(), }) .await?; let mut tree_changes = vec![]; + let mut updates = vec![]; + let mut expected_state_version = None; // syncing states while let Some(result) = state_stream.next().await { - let msg = match result { - Ok(msg) => msg, - Err(err) if err.is_not_found() => { - return Ok(maybe_persisted_state_version); - }, - Err(err) => { - return Err(err.into()); - }, - }; + let msg = result?; - if msg.transitions.is_empty() { + if msg.updates.is_empty() { return Err(RpcStateSyncError::InvalidResponse(anyhow!( "Received empty state transition batch." ))); } - if msg.transitions.len() > STATE_SYNC_MAX_BATCH_SIZE { + if msg.updates.len() > STATE_SYNC_MAX_BATCH_SIZE { return Err(RpcStateSyncError::InvalidResponse(anyhow!( - "Received too many state transitions in a batch: {}. Expected at most {}.", - msg.transitions.len(), + "Received too many state updates in a batch: {}. Expected at most {}.", + msg.updates.len(), STATE_SYNC_MAX_BATCH_SIZE ))); } + if msg.state_version < start_state_version { + return Err(RpcStateSyncError::InvalidResponse(anyhow!( + "Received state version {} that is less than the persisted state version {}.", + msg.state_version, + start_state_version + ))); + } - self.stats.total_transitions += msg.transitions.len() as u64; - - // Batch the response into a state_version -> Vec map - let mut transitions = - msg.transitions - .into_iter() - .try_fold(BTreeMap::<_, Vec<_>>::new(), |mut acc, transition| { - let transition = - StateTransition::try_from(transition).map_err(RpcStateSyncError::InvalidResponse)?; - acc.entry(transition.state_version).or_default().push(transition); - Ok::<_, RpcStateSyncError>(acc) - })?; - - loop { - let Some((state_version, transitions_for_state_version)) = transitions.pop_first() else { - info!(target: LOG_TARGET, "🛜 No more state transitions to process for shard {shard}"); - break; - }; - - tree_changes.reserve_exact(transitions_for_state_version.len()); - - self.state_store.with_write_tx(|tx| { - info!( - target: LOG_TARGET, - "🛜 Next state updates batch of size {} from v{}", - transitions_for_state_version.len(), - state_version - ); + if expected_state_version.is_some_and(|v| v != msg.state_version) { + return Err(RpcStateSyncError::InvalidResponse(anyhow!( + "Received state version {} that is not the expected state version {}.", + msg.state_version, + expected_state_version.unwrap() + ))); + } - let mut store = ShardScopedTreeStoreWriter::new(tx, shard); + self.stats.total_transitions += msg.updates.len() as u64; + + tree_changes.reserve_exact(msg.updates.len()); + updates.reserve_exact(msg.updates.len()); + + let state_version = msg.state_version; + let updates_for_state_version = msg + .updates + .into_iter() + .map(|t| SubstateUpdateProof::try_from(t).map_err(RpcStateSyncError::InvalidResponse)); + let msg_epoch = msg.epoch.map(Epoch::from).ok_or_else(|| { + RpcStateSyncError::InvalidResponse(anyhow!("Received state transition with no epoch")) + })?; + + info!(target: LOG_TARGET, "🛜 Buffering {} state update(s) (state version: v{})", updates_for_state_version.len(), state_version); + for result in updates_for_state_version { + let update = result?; + let (tree_change, template_change) = extract_tree_change(msg_epoch, &update)?; + + debug!(target: LOG_TARGET, "🛜 -> state update (v{}) {}", state_version, update); + template_changes.extend(template_change); + tree_changes.push(tree_change); + updates.push(update); + } - for transition in transitions_for_state_version { - if transition.id.shard() != shard { - return Err(RpcStateSyncError::InvalidResponse(anyhow!( - "Received state transition for shard {} which is not the expected shard {}.", - transition.id.shard(), - shard - ))); - } + info!(target: LOG_TARGET, "🛜 Sync: {} state update(s), {} new template(s) (state version: v{})", updates.len(), template_changes.len(), state_version); - if transition.id.epoch().is_zero() { - return Err(RpcStateSyncError::InvalidResponse(anyhow!( - "Received state transition with epoch 0." - ))); - } + if msg.has_more { + info!( + target: LOG_TARGET, + "🛜 Received more state updates for v{}. Continuing to buffer...", + state_version + ); + // Continue buffering + // TODO: maximum possible state transitions within a single state version? + expected_state_version = Some(state_version); + continue; + } - if transition.id.epoch() >= current_epoch { - return Err(RpcStateSyncError::InvalidResponse(anyhow!( - "Received state transition for epoch {} which is at or ahead of our current epoch {}.", - transition.id.epoch(), - current_epoch - ))); - } + expected_state_version = None; - let change = match &transition.update { - SubstateUpdate::Create(create) => { - let id = create.substate.as_versioned_substate_id_ref(); - if let Some(template_address) = create.substate.substate_id.as_template() { - match create - .substate - .value - .value() { - Some(value) => { - let template = value.as_template() - .ok_or_else(|| RpcStateSyncError::InvalidResponse( - anyhow!("Validator returned a template address {} but substate value was not a template", id.substate_id()) - ))?; - - info!(target: LOG_TARGET, "🛜 Add template {id}"); - template_changes_mut.push(TemplateChange::Add { - template_address, - author_public_key: template.author, - binary_hash: template.binary_hash.into_array().into(), - epoch: transition.id.epoch(), - }); - } - None => { - // TODO: currently you cannot DOWN a template. If we were to allow deprecations, it would likely be marking the template as deprecated rather than DOWNing it, and not permitting any template (non-component) calls to the template. - // We could still handle this case by requesting the template by address and verifying the template address hash i.e. peers send author and binary. - warn!(target: LOG_TARGET, "❗️ NEVER HAPPEN: Validator sent us a template {} that has no value, indicating it will be DOWNed later. We are not able to sync it", id); - } - }; - } - - SubstateTreeChange::Up { - id: id.to_owned(), - value_hash: create.substate.to_value_hash(), - } - } - SubstateUpdate::Destroy(destroy) => { - if let Some(template_address) = destroy.substate_id.as_template() { - info!(target: LOG_TARGET, "🛜 Deprecate template {}", template_address); - template_changes_mut.push(TemplateChange::Deprecate { template_address }); - } - - SubstateTreeChange::Down { - id: destroy.to_versioned_substate_id() - } - } - }; - - info!(target: LOG_TARGET, "🛜 Applying state update (v{}) {}", state_version, transition); - self.commit_update(store.transaction(), checkpoint, checkpoint_block_id, transition)?; - - tree_changes.push(change); - } + // Verify and commit changes + self.state_store.with_write_tx(|tx| { + info!( + target: LOG_TARGET, + "🛜 Next state updates batch of size {} from v{}", + updates.len(), + state_version + ); - info!(target: LOG_TARGET, "🛜 {} state update(s) for v{}", tree_changes.len(), state_version); + let mut store = ShardScopedTreeStoreWriter::new(tx, shard); + + info!(target: LOG_TARGET, "🛜 {} state update(s) for v{}", updates.len(), state_version); + self.commit_updates( + store.transaction(), + shard, + msg_epoch, + msg.state_version, + updates.drain(..), + )?; + + // Persist tree changes + if !tree_changes.is_empty() { + let mut state_tree = SpreadPrefixStateTree::new(&mut store); + info!(target: LOG_TARGET, "🛜 Committing {} state tree changes batch v{}", tree_changes.len(), state_version); + let local_state_root = state_tree.batch_put_substate_changes(maybe_persisted_state_version, state_version, tree_changes.drain(..))?; + // Only check the state root once we have reached the checkpoint state version + // TODO: we should sync to multiple checkpoints to catch misbehaviour earlier + if state_version == checkpoint_state_version { + if local_state_root != checkpoint_shard_root { + error!( + target: LOG_TARGET, + "❌ State root mismatch for {shard}. Checkpoint {expected} but got {actual}. Rolling back.", + expected = checkpoint_shard_root, + actual = local_state_root, + ); + + // rollback! + return Err(RpcStateSyncError::StateRootMismatch { + expected: checkpoint_shard_root, + actual: local_state_root, + }); + } + info!( + target: LOG_TARGET, + "🛜 ✅ State root for {shard} matches checkpoint: {local_state_root} (v{state_version})", + ); - if !tree_changes.is_empty() { - let mut state_tree = SpreadPrefixStateTree::new(&mut store); - info!(target: LOG_TARGET, "🛜 Committing {} state tree changes batch v{}", tree_changes.len(), state_version); - state_tree.batch_put_substate_changes(maybe_persisted_state_version, state_version, tree_changes.drain(..))?; maybe_persisted_state_version = Some(state_version); - store.set_version(state_version)?; + store.set_state_version(state_version)?; + // Done + return Ok(()); } - Ok::<_, RpcStateSyncError>(()) - })?; - } - } + maybe_persisted_state_version = Some(state_version); + store.set_state_version(state_version)?; + } - let local_state_root = self.calculate_state_root_for_shard(shard, maybe_persisted_state_version)?; - if local_state_root != checkpoint_state_root { - error!( - target: LOG_TARGET, - "❌State root mismatch for {shard}. Checkpoint {expected} but got {actual}. Rolling back.", - expected = checkpoint_state_root, - actual = local_state_root, - ); - - // TODO: rollback - return Err(RpcStateSyncError::StateRootMismatch { - expected: checkpoint_state_root, - actual: local_state_root, - }); + Ok::<_, RpcStateSyncError>(()) + })?; } - info!(target: LOG_TARGET, "🛜 Synced state for {shard} to v{} with root {local_state_root}", maybe_persisted_state_version.unwrap_or(0)); + info!(target: LOG_TARGET, "🛜 Synced state for {shard} to v{}", maybe_persisted_state_version.unwrap_or(1)); - Ok(maybe_persisted_state_version) + Ok((maybe_persisted_state_version, template_changes)) } fn calculate_state_root_for_shard( &self, + tx: &::ReadTransaction<'_>, shard: Shard, version: Option, ) -> Result { let Some(version) = version else { return Ok(SPARSE_MERKLE_PLACEHOLDER_HASH); }; - self.state_store.with_read_tx(|tx| { - let mut store = ShardScopedTreeStoreReader::new(tx, shard); - let state_tree = SpreadPrefixStateTree::new(&mut store); - let root = state_tree.get_root_hash(version)?; - Ok(root) - }) + let mut store = ShardScopedTreeStoreReader::new(tx, shard); + let state_tree = SpreadPrefixStateTree::new(&mut store); + let root = state_tree.get_root_hash(version)?; + Ok(root) } - pub fn commit_update( + pub fn commit_updates>( &self, tx: &mut TTx, - checkpoint: &EpochCheckpoint, - checkpoint_block_id: BlockId, - transition: StateTransition, + shard: Shard, + epoch: Epoch, + state_version: Version, + updates: I, ) -> Result<(), StorageError> { - match transition.update { - SubstateUpdate::Create(SubstateCreatedProof { substate }) => { - SubstateRecord::new( - substate.substate_id, - substate.version, - substate.value, - transition.id.shard(), - transition.id.epoch(), - checkpoint_block_id, - // TODO: correct QC ID - QcId::zero(), - ) - .create(tx)?; - }, - SubstateUpdate::Destroy(SubstateDestroyedProof { substate_id, version }) => { - SubstateRecord::destroy( - tx, - VersionedSubstateId::new(substate_id, version), - transition.id.shard(), - transition.id.epoch(), - checkpoint.header().height.into(), - // TODO - &QcId::zero(), - )?; - }, - } + let batch_updates = IndexMap::from_iter([(shard, SubstateTransitionData { + state_version, + transitions: updates + .into_iter() + .map(|update| match update { + SubstateUpdateProof::Create(create) => SubstateTransition::Up { + id: create.substate.substate_id, + version: create.substate.version, + substate_or_hash: create.substate.value, + }, + SubstateUpdateProof::Destroy(destroy) => SubstateTransition::Down { + id: VersionedSubstateId::new(destroy.substate_id, destroy.version), + }, + }) + .collect(), + })]); + let batch = SubstateUpdateBatch { + epoch, + updates: batch_updates, + }; + + SubstateRecord::commit_batch(tx, batch)?; Ok(()) } @@ -523,13 +469,15 @@ where TConsensusSpec: ConsensusSpec }, }; - let mut template_changes = vec![]; + let maybe_persisted_state_version = self + .state_store + .with_read_tx(|tx| tx.state_tree_versions_get_latest(shard))?; match self - .start_state_sync(&mut client, shard, &checkpoint, &mut template_changes) + .start_state_sync(&mut client, shard, &checkpoint, maybe_persisted_state_version) .await { - Ok(maybe_version) => { + Ok((maybe_version, template_changes)) => { // We only enqueue these if state sync succeeds and the state root matches if !template_changes.is_empty() { self.template_manager.enqueue_template_changes(template_changes).await?; @@ -628,18 +576,13 @@ where TConsensusSpec: ConsensusSpec let local_shard_group = local_info.shard_group(); - let mut shard_state_roots = IndexMap::with_capacity(local_shard_group.len() + 1); - - let maybe_version = self - .sync_global_shard( - current_epoch, - ShardGroup::all_shards(local_info.num_preshards()), - &prev_epoch_committees, - &our_vn.address, - ) - .await?; - let local_state_root = self.calculate_state_root_for_shard(Shard::global(), maybe_version)?; - shard_state_roots.insert(Shard::global(), local_state_root); + self.sync_global_shard( + current_epoch, + ShardGroup::all_shards(local_info.num_preshards()), + &prev_epoch_committees, + &our_vn.address, + ) + .await?; // Sync data from each committee in range of the committee we're joining. // NOTE: we don't have to worry about substates in address range because shard boundaries are fixed. @@ -652,19 +595,11 @@ where TConsensusSpec: ConsensusSpec continue; }; for shard in intersect_shard_group.shard_iter() { - let maybe_current_version = self - .sync_shard(shard, shard_group, current_epoch, &committee, &our_vn.address) + self.sync_shard(shard, shard_group, current_epoch, &committee, &our_vn.address) .await?; - let local_state_root = self.calculate_state_root_for_shard(shard, maybe_current_version)?; - shard_state_roots.insert_sorted(shard, local_state_root); } } - // Calculate the shard group merkle root and save it for the next genesis - let final_state_root = compute_merkle_root_for_hashes(shard_state_roots.into_values())?; - self.state_store - .with_write_tx(|tx| EpochStateRoot::new(current_epoch, local_shard_group, final_state_root).set(tx))?; - self.stats.total_time = timer.elapsed(); Ok(()) } @@ -693,17 +628,89 @@ where TConsensusSpec: ConsensusSpec + Send + Sync + 'static async fn sync(&mut self) -> Result<(), Self::Error> { if let Err(err) = self.sync_inner().await { - warn!(target: LOG_TARGET, "🛜State sync failed: {err}"); + warn!(target: LOG_TARGET, "🛜State sync failed: {err} (stats: {})", self.stats); // Clear the valid checkpoints cache self.valid_checkpoints = HashMap::new(); + self.stats = StateSyncStats::default(); return Err(err); } + info!(target: LOG_TARGET, "🛜State sync completed successfully: {}", self.stats); + // Clear the valid checkpoints cache self.valid_checkpoints = HashMap::new(); - - info!(target: LOG_TARGET, "🛜State sync complete: {}", self.stats); self.stats = StateSyncStats::default(); Ok(()) } } + +fn extract_template_change( + // Extra data required by the template db - necessary? + epoch: Epoch, + create: &SubstateCreatedProof, +) -> Result, RpcStateSyncError> { + let Some(template_address) = create.substate.substate_id.as_template() else { + return Ok(None); + }; + match create.substate.value.value() { + Some(value) => { + let template = value.as_template().ok_or_else(|| { + // This is possible if the VN is malicious + RpcStateSyncError::InvalidResponse(anyhow!( + "Validator returned a template address {} but substate value was not a template", + create.substate.substate_id() + )) + })?; + + info!(target: LOG_TARGET, "🛜 Add template {}", create.substate.substate_id); + Ok(Some(TemplateChange::Add { + template_address, + author_public_key: template.author, + binary_hash: template.binary_hash.into_array().into(), + epoch, + })) + }, + None => { + // TODO: currently you cannot DOWN a template. If we were to allow deprecations, it would likely be + // marking the template as deprecated rather than DOWNing it, and not permitting any template + // (non-component) calls to the template. We could still handle this case by requesting + // the template by address and verifying the template address hash i.e. peers send author and binary. + warn!(target: LOG_TARGET, "❗️ NEVER HAPPEN: Validator sent us a template {} that has no value, indicating it was DOWNed later. We are not able to sync it", create.substate.substate_id); + Ok(None) + }, + } +} + +fn extract_tree_change( + epoch: Epoch, + update: &SubstateUpdateProof, +) -> Result<(SubstateTreeChange, Option), RpcStateSyncError> { + match update { + SubstateUpdateProof::Create(create) => { + let id = create.substate.as_versioned_substate_id_ref(); + let template_change = extract_template_change(epoch, create)?; + + Ok(( + SubstateTreeChange::Up { + id: id.to_owned(), + value_hash: create.substate.to_value_hash(), + }, + template_change, + )) + }, + SubstateUpdateProof::Destroy(destroy) => { + let template_change = destroy.substate_id.as_template().map(|template_address| { + // TODO: Currently not possible to down a template + info!(target: LOG_TARGET, "🛜 Deprecate template {}", template_address); + TemplateChange::Deprecate { template_address } + }); + + Ok(( + SubstateTreeChange::Down { + id: destroy.to_versioned_substate_id(), + }, + template_change, + )) + }, + } +} diff --git a/crates/state_store_rocksdb/src/cf_api.rs b/crates/state_store_rocksdb/src/cf_api.rs index e478d0f49c..66df36d84e 100644 --- a/crates/state_store_rocksdb/src/cf_api.rs +++ b/crates/state_store_rocksdb/src/cf_api.rs @@ -15,6 +15,7 @@ use tari_ootle_storage::Ordering; use crate::{ codecs::{DbCodec, EncodeVec, UnitCodec}, error::RocksDbStorageError, + range::QueryRange, traits::{Cf, QueryCf, RocksReader, RocksWriter}, }; @@ -162,7 +163,7 @@ impl CfContext<'_, DB, CF> { } let keys = keys.map(|k| { - // We don't support key encoding failing here for mem allocation reasons. Generally key encoding is + // We don't support key encoding failing here for mem allocation reasons. Generally, key encoding is // infallible so we should evaluate whether to change the codec to be infallible. If key encoding on the // database level ever fails a crash is reasonable. let key = self.key_codec.encode(k.borrow()).expect("Failed to encode key"); @@ -428,7 +429,7 @@ impl CfContext<'_, DB, TQuery> { prefix: &TQuery::Key, ) -> impl Iterator::Cf as Cf>::Value, RocksDbStorageError>> + 'a { let key = self.encode_key(prefix); - let iter = self.range_iterator_with_codecs::::Cf as Cf>::ValueCodec, (), <::Cf as Cf>::Value, >( ordering, rocksdb::PrefixRange(key) ); + let iter = self.range_iterator_with_codecs::::Cf as Cf>::ValueCodec, (), <::Cf as Cf>::Value, >(ordering, rocksdb::PrefixRange(key)); iter.map(|res| { let (_, v) = res?; Ok::<_, RocksDbStorageError>(v) @@ -452,6 +453,46 @@ impl CfContext<'_, DB, TQuery> { }) } + /// Returns a decoded key value iterator over the range of keys (exclusive). + pub fn query_range_iterator, R: Into>>( + &self, + ordering: Ordering, + range: R, + ) -> impl Iterator, RocksDbStorageError>> + '_ { + let iter: Box> = match range.into() { + QueryRange::Exclusive { start, end } => { + let start = self.encode_key(start.borrow()); + let end = self.encode_key(end.borrow()); + Box::new(self.range_iterator_with_codecs::< + ::KeyCodec, + ::ValueCodec, + ::Key, + ::Value + >(ordering, start..end)) + }, + QueryRange::From { start } => { + let start = self.encode_key(start.borrow()); + Box::new(self.range_iterator_with_codecs::< + ::KeyCodec, + ::ValueCodec, + ::Key, + ::Value + >(ordering, start..)) + }, + QueryRange::To { end } => { + let end = self.encode_key(end.borrow()); + Box::new(self.range_iterator_with_codecs::< + ::KeyCodec, + ::ValueCodec, + ::Key, + ::Value + >(ordering, ..end)) + }, + }; + + iter + } + /// Returns an iterator over the range of keys (exclusive). pub fn query_range_key_iterator>( &self, diff --git a/crates/state_store_rocksdb/src/codecs/mod.rs b/crates/state_store_rocksdb/src/codecs/mod.rs index 181af60745..652d605334 100644 --- a/crates/state_store_rocksdb/src/codecs/mod.rs +++ b/crates/state_store_rocksdb/src/codecs/mod.rs @@ -38,5 +38,5 @@ pub trait DbCodec { } pub type DefaultCodec = Bincode; -pub type DefaultVersionedCodec = VersionedCodec, T>; +pub type DefaultVersionedCodec = VersionedCodec, T>; pub type DefaultCodecRef = BincodeRef; diff --git a/crates/state_store_rocksdb/src/codecs/tuple.rs b/crates/state_store_rocksdb/src/codecs/tuple.rs index f5087fa126..98583df4d9 100644 --- a/crates/state_store_rocksdb/src/codecs/tuple.rs +++ b/crates/state_store_rocksdb/src/codecs/tuple.rs @@ -4,12 +4,12 @@ use anyhow::anyhow; use tari_consensus_types::BlockId; use tari_ootle_common_types::{shard::Shard, Epoch, NodeHeight}; -use tari_ootle_storage::consensus_models::{ForeignProposalStatus, StateTransitionId}; +use tari_ootle_storage::consensus_models::ForeignProposalStatus; use tari_template_lib_types::crypto::RistrettoPublicKeyBytes; use tari_transaction::TransactionId; use crate::{ - codecs::{Column, DbCodec, EncodeVec, EpochCodec, NumberCodec, ShardCodec}, + codecs::{Column, DbCodec, EncodeVec}, error::RocksDbStorageError, }; @@ -143,34 +143,6 @@ where } } -/// Encodes in (A, B, C) order. (Shard, Seq, Epoch) and (Epoch, Shard, Seq) keys are supported. -#[derive(Default)] -pub struct StateTransitionIdCodec { - codec: (A, B, C), -} - -impl DbCodec for StateTransitionIdCodec, EpochCodec> { - fn encode(&self, id: &StateTransitionId) -> Result { - self.codec.encode(&(id.shard(), id.seq(), id.epoch())) - } - - fn decode(&self, bytes: &[u8]) -> Result { - let (shard, seq, epoch) = self.codec.decode(bytes)?; - Ok(StateTransitionId::new(epoch, shard, seq)) - } -} - -impl DbCodec for StateTransitionIdCodec> { - fn encode(&self, id: &StateTransitionId) -> Result { - self.codec.encode(&(id.epoch(), id.shard(), id.seq())) - } - - fn decode(&self, bytes: &[u8]) -> Result { - let (epoch, shard, seq) = self.codec.decode(bytes)?; - Ok(StateTransitionId::new(epoch, shard, seq)) - } -} - pub trait FixedByteLength { const BYTE_LENGTH: usize; } diff --git a/crates/state_store_rocksdb/src/column_families/block_diff.rs b/crates/state_store_rocksdb/src/column_families/block_diff.rs index 817a26b702..13aa1666a8 100644 --- a/crates/state_store_rocksdb/src/column_families/block_diff.rs +++ b/crates/state_store_rocksdb/src/column_families/block_diff.rs @@ -38,8 +38,8 @@ pub struct BlockDiffKey { pub substate_id: SubstateId, pub version: u32, pub is_up: bool, - /// Retains the ordering of the substate changes in the block. This limits the mximum number of substate changes in - /// a block to u32::MAX (4,294,967,295). + /// Retains the ordering of the substate changes in the block. This limits the maximum number of substate changes + /// in a block to u32::MAX (4,294,967,295). pub sequence: u32, } diff --git a/crates/state_store_rocksdb/src/column_families/bookkeeping.rs b/crates/state_store_rocksdb/src/column_families/bookkeeping.rs index d629960d2a..f8262fe8d3 100644 --- a/crates/state_store_rocksdb/src/column_families/bookkeeping.rs +++ b/crates/state_store_rocksdb/src/column_families/bookkeeping.rs @@ -16,7 +16,6 @@ use tari_consensus_types::{ LockedBlock, }; use tari_ootle_common_types::NodeHeight; -use tari_ootle_storage::consensus_models::EpochStateRoot; use crate::{ codecs::{ByteColumn, ColumnCodec, DefaultCodec, NumberCodec}, @@ -46,9 +45,6 @@ enum BookKeepingKey { HighQc, /// The last high timeout certificate HighTc, - /// The state root of the previous epoch. This is set based on either calculated root of the current shard group - /// after a successful sync, or based on the last checkpoint - PreviousEpochStateRoot, /// The highest block seen by the node HighestSeenBlock, /// The last sent new view message @@ -68,9 +64,8 @@ impl BookKeepingKey { Self::LeafBlock => 7, Self::HighQc => 8, Self::HighTc => 9, - Self::PreviousEpochStateRoot => 10, - Self::HighestSeenBlock => 11, - Self::LastSentNewView => 12, + Self::HighestSeenBlock => 10, + Self::LastSentNewView => 11, } } } @@ -212,19 +207,6 @@ impl Cf for HighTcCf { } } -pub struct PreviousEpochStateRootCf; - -impl Cf for PreviousEpochStateRootCf { - type Key = ByteColumn<{ BookKeepingKey::PreviousEpochStateRoot.as_byte() }>; - type KeyCodec = ColumnCodec; - type Value = EpochStateRoot; - type ValueCodec = DefaultCodec; - - fn name() -> &'static str { - CF_NAME - } -} - pub struct HighestSeenBlockCf; impl Cf for HighestSeenBlockCf { diff --git a/crates/state_store_rocksdb/src/column_families/state_transition.rs b/crates/state_store_rocksdb/src/column_families/state_transition.rs index 99abffa6cc..ee0a7376f2 100644 --- a/crates/state_store_rocksdb/src/column_families/state_transition.rs +++ b/crates/state_store_rocksdb/src/column_families/state_transition.rs @@ -21,20 +21,51 @@ // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. use serde::{Deserialize, Serialize}; -use tari_ootle_common_types::{shard::Shard, SubstateAddress}; -use tari_ootle_storage::consensus_models::StateTransitionId; +use tari_ootle_common_types::{shard::Shard, Epoch, SubstateAddress}; use tari_state_tree::Version; use crate::{ - codecs::{DefaultCodec, EpochCodec, NumberCodec, ShardCodec, StateTransitionIdCodec}, - traits::{Cf, QueryCf}, + codecs::{DefaultVersionedCodec, NumberCodec, ShardCodec}, + traits::{Cf, QueryCf, Versioned}, }; #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StateTransitionModelData { +pub struct StateTransitionModelDataV1 { + pub epoch: Epoch, + pub transitions: Vec, + pub state_version: Version, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum VersionedStateTransitionModelData { + V1(StateTransitionModelDataV1), +} + +impl Versioned for VersionedStateTransitionModelData { + type Latest = StateTransitionModelDataV1; + + fn upgrade_single_step(self) -> (Self, bool) { + match self { + Self::V1(_) => (self, false), // No upgrades available + } + } + + fn into_latest(self) -> Self::Latest { + match self { + Self::V1(record) => record, + } + } +} + +impl From for VersionedStateTransitionModelData { + fn from(record: StateTransitionModelDataV1) -> Self { + Self::V1(record) + } +} +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StateTransitionRecordData { pub substate_address: SubstateAddress, pub transition: StateTransitionType, - pub state_version: Version, } #[derive(Debug, Clone, Copy, Serialize, Deserialize)] @@ -46,41 +77,20 @@ pub enum StateTransitionType { pub struct StateTransitionCf; impl Cf for StateTransitionCf { - type Key = StateTransitionId; - type KeyCodec = StateTransitionIdCodec, EpochCodec>; - type Value = StateTransitionModelData; - type ValueCodec = DefaultCodec; + type Key = (Shard, Version); + type KeyCodec = (ShardCodec, NumberCodec); + type Value = StateTransitionModelDataV1; + type ValueCodec = DefaultVersionedCodec; fn name() -> &'static str { "state_transitions" } } -pub struct ByShardQuery; - -impl QueryCf for ByShardQuery { - type Cf = StateTransitionCf; - type Key = Shard; - type KeyCodec = ShardCodec; -} - -pub struct ByShardAndIdQuery; +pub struct ByShardAndStateVersionQuery; -impl QueryCf for ByShardAndIdQuery { +impl QueryCf for ByShardAndStateVersionQuery { type Cf = StateTransitionCf; - type Key = (Shard, u64); - type KeyCodec = (ShardCodec, NumberCodec); -} - -pub struct ShardSeqIndex; - -impl Cf for ShardSeqIndex { - type Key = Shard; - type KeyCodec = ShardCodec; - type Value = u64; - type ValueCodec = NumberCodec; - - fn name() -> &'static str { - "state_transition_shard_seq_idx" - } + type Key = (Shard, Version); + type KeyCodec = (ShardCodec, NumberCodec); } diff --git a/crates/state_store_rocksdb/src/column_families/state_tree.rs b/crates/state_store_rocksdb/src/column_families/state_tree.rs index 98d890712e..d767943aee 100644 --- a/crates/state_store_rocksdb/src/column_families/state_tree.rs +++ b/crates/state_store_rocksdb/src/column_families/state_tree.rs @@ -21,7 +21,7 @@ // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. use tari_ootle_common_types::shard::Shard; -use tari_state_tree::{Node, NodeKey, StaleTreeNode, Version}; +use tari_state_tree::{Node, NodeKey, StaleTreeNode, StateTreePayload, Version}; use crate::{ codecs::{DefaultCodec, NodeKeyCodec, NumberCodec, ShardCodec}, @@ -33,7 +33,7 @@ pub struct StateTreeCf; impl Cf for StateTreeCf { type Key = (Shard, NodeKey); type KeyCodec = (ShardCodec, NodeKeyCodec); - type Value = Node; + type Value = Node; type ValueCodec = DefaultCodec; fn name() -> &'static str { @@ -48,7 +48,7 @@ pub struct StateTreeCfRef<'a> { impl<'a> Cf for StateTreeCfRef<'a> { type Key = (Shard, &'a NodeKey); type KeyCodec = (ShardCodec, NodeKeyCodec); - type Value = Node; + type Value = Node; type ValueCodec = DefaultCodec; fn name() -> &'static str { @@ -64,12 +64,13 @@ impl Default for StateTreeCfRef<'_> { } } -pub struct ByShardQuery; +pub struct ByShardStateVersionQuery; -impl QueryCf for ByShardQuery { +impl QueryCf for ByShardStateVersionQuery { type Cf = StateTreeCf; - type Key = Shard; - type KeyCodec = ShardCodec; + type Key = (Shard, Version); + // Depends on NodeKeyCodec first serializing the Shard, then the Version. + type KeyCodec = (ShardCodec, NumberCodec); } pub struct StateTreeStaleNodesModel; diff --git a/crates/state_store_rocksdb/src/column_families/state_tree_shard_versions.rs b/crates/state_store_rocksdb/src/column_families/state_tree_shard_versions.rs index 9a3e739940..0b6cf2f2db 100644 --- a/crates/state_store_rocksdb/src/column_families/state_tree_shard_versions.rs +++ b/crates/state_store_rocksdb/src/column_families/state_tree_shard_versions.rs @@ -25,7 +25,7 @@ use tari_state_tree::Version; use crate::{ codecs::{NumberCodec, ShardCodec}, - traits::Cf, + traits::{Cf, QueryCf}, }; pub struct StateTreeShardVersionCf; @@ -40,3 +40,11 @@ impl Cf for StateTreeShardVersionCf { "state_tree_shard_versions" } } + +pub struct ByShard; + +impl QueryCf for ByShard { + type Cf = StateTreeShardVersionCf; + type Key = Shard; + type KeyCodec = ShardCodec; +} diff --git a/crates/state_store_rocksdb/src/dbs/transaction.rs b/crates/state_store_rocksdb/src/dbs/transaction.rs index 9f9cc2b3f3..3eb49c92fc 100644 --- a/crates/state_store_rocksdb/src/dbs/transaction.rs +++ b/crates/state_store_rocksdb/src/dbs/transaction.rs @@ -53,7 +53,11 @@ impl RocksReader for Transaction<'_, TransactionDB> { I: IntoIterator, W: 'b + AsColumnFamilyRef, { - self.multi_get_cf(keys) + let mut read_opts = ReadOptions::default(); + // We always use bulk scans with multi_get, so we disable cache to avoid unnecessary overhead. + read_opts.fill_cache(false); + read_opts.set_verify_checksums(false); + self.multi_get_cf_opt(keys, &read_opts) } } diff --git a/crates/state_store_rocksdb/src/lib.rs b/crates/state_store_rocksdb/src/lib.rs index fcd06cfaab..32b6829a01 100644 --- a/crates/state_store_rocksdb/src/lib.rs +++ b/crates/state_store_rocksdb/src/lib.rs @@ -39,6 +39,7 @@ pub mod snapshot; mod options; pub use options::*; +mod range; mod read_only; #[cfg(test)] mod tests; diff --git a/crates/state_store_rocksdb/src/range.rs b/crates/state_store_rocksdb/src/range.rs new file mode 100644 index 0000000000..97fa9d018b --- /dev/null +++ b/crates/state_store_rocksdb/src/range.rs @@ -0,0 +1,35 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use std::ops::{Range, RangeFrom, RangeTo}; + +/// A subset of RangeBounds that are possible to query in RocksDB. +pub enum QueryRange { + // start..end + Exclusive { start: B, end: B }, + // start.. + From { start: B }, + // ..end + To { end: B }, +} + +impl From> for QueryRange { + fn from(range: Range) -> Self { + QueryRange::Exclusive { + start: range.start, + end: range.end, + } + } +} + +impl From> for QueryRange { + fn from(range: RangeFrom) -> Self { + QueryRange::From { start: range.start } + } +} + +impl From> for QueryRange { + fn from(range: RangeTo) -> Self { + QueryRange::To { end: range.end } + } +} diff --git a/crates/state_store_rocksdb/src/reader.rs b/crates/state_store_rocksdb/src/reader.rs index d54378909d..3addfa4124 100644 --- a/crates/state_store_rocksdb/src/reader.rs +++ b/crates/state_store_rocksdb/src/reader.rs @@ -57,6 +57,8 @@ use tari_ootle_common_types::{ Epoch, NodeAddressable, NodeHeight, + ShardGroup, + ShardStateVersions, SubstateAddress, ToSubstateAddress, VersionedSubstateIdRef, @@ -67,12 +69,10 @@ use tari_ootle_storage::{ BlockDiff, BlockTransactionExecution, EpochCheckpoint, - EpochStateRoot, ForeignProposalRecord, LockedSubstateValue, PendingShardStateTreeDiff, - StateTransition, - StateTransitionId, + StateVersionTransitions, SubstateChange, SubstateCreatedProof, SubstateData, @@ -80,7 +80,7 @@ use tari_ootle_storage::{ SubstateLock, SubstatePledges, SubstateRecord, - SubstateUpdate, + SubstateUpdateProof, SubstateValueOrHash, TransactionExecution, TransactionPoolRecord, @@ -93,7 +93,7 @@ use tari_ootle_storage::{ StateStoreReadTransaction, StorageError, }; -use tari_state_tree::{Node, NodeKey, Version}; +use tari_state_tree::{Node, NodeKey, StateTreePayload, Version}; use tari_template_lib_types::crypto::RistrettoPublicKeyBytes; use tari_transaction::TransactionId; @@ -119,7 +119,6 @@ use crate::{ LastVotedCf, LeafBlockCf, LockedBlockCf, - PreviousEpochStateRootCf, }, burnt_utxo, burnt_utxo::BurntUtxoCf, @@ -137,8 +136,10 @@ use crate::{ lock_conflict, pending_state_tree_diff, state_transition, - state_transition::{StateTransitionCf, StateTransitionType}, + state_transition::StateTransitionType, + state_tree, state_tree::StateTreeCfRef, + state_tree_shard_versions, state_tree_shard_versions::StateTreeShardVersionCf, substate, substate::SubstateCf, @@ -189,7 +190,7 @@ impl<'a, TAddr> RocksDbStateStoreReadTransaction<'a, TAddr> { impl<'a, TAddr: NodeAddressable + Serialize + DeserializeOwned + 'a> RocksDbStateStoreReadTransaction<'a, TAddr> { /// Returns the blocks until the end_block (inclusive). NOTE: there is no specific order in the returned blocks /// (HashSet) so this should only be used to determine ex/inclusion in the set. The end_block should be a block - /// in the pending chain, if not an empty list is returned. + /// in the pending chain if not an empty list is returned. fn get_pending_chain_until(&self, end_block: &BlockId) -> Result, RocksDbStorageError> { const OPERATION: &str = "get_pending_chain_until"; trace!(target: LOG_TARGET, "{OPERATION}: end: {end_block}"); @@ -1588,51 +1589,47 @@ impl<'tx, TAddr: NodeAddressable + Serialize + DeserializeOwned + 'tx> StateStor Ok(diffs) } - fn state_transitions_get_n_after( + fn state_transitions_get_after( &self, - n: usize, - id: StateTransitionId, - end_epoch: Epoch, - ) -> Result, StorageError> { + req_shard: Shard, + state_version: Version, + include_values: bool, + ) -> Result { const OPERATION: &str = "state_transitions_get_n_after"; - // The StateTransitionId may not exist and is used to find subsequent state transitions - let cf = self.db().cf(StateTransitionCf)?; - let query = self.db().cf(state_transition::ByShardAndIdQuery)?; - let iter = query.query_start_range_key_iterator(Ordering::Ascending, &(id.shard(), id.seq() + 1)); + let query = self.db().cf(state_transition::ByShardAndStateVersionQuery)?; + let mut iter = query.query_range_iterator(Ordering::Ascending, (req_shard, state_version)..); let substate_cf = self.db().cf(SubstateCf)?; - let mut transitions = Vec::with_capacity(n); - // TODO: this loads and searches a lot of keys which are not applicable to the end epoch. We'll need to use an - // epoch prefixed index (maybe only tracking the last transition with a (shard, epoch) key), or figure out some - // other way to get state transitions (e.g can we iterate the JMT?) - for result in iter { - let key = result?; - - if key.shard() > id.shard() { - // We're done when we move to the next shard - break; - } - - if key.epoch() >= end_epoch { - // We are not ordering by Epoch, so subsequent epochs could be in range, so we have to continue. - // TODO(perf): consider an epoch ordered index - continue; - } + // NOTE: The state version may not have any transitions + let result = iter.next().ok_or_else(|| StorageError::NotFound { + item: "StateTransition", + key: format!("shard {} and state version {}", req_shard, state_version), + })?; + let ((shard, version), data) = result?; - // We could also get this from the iterator - if this doesn't require a drive seek then it seems better to - // only deserialize when needed - let value = cf.get(&key, OPERATION)?; + // If we've scanned onto the next shard, we couldn't find the requested shard + if shard != req_shard { + return Err(StorageError::NotFound { + item: "StateTransition", + key: format!("shard {} and state version {}", req_shard, state_version), + }); + } - let substate = substate_cf.get(&value.substate_address, OPERATION)?; + // TODO(perf): if include_values is false, we still have to load the whole substate for the id and version - not + // ideal + let substates = substate_cf.multi_get(data.transitions.iter().map(|t| t.substate_address), OPERATION)?; - let update = match value.transition { + let mut updates = Vec::with_capacity(data.transitions.len()); + // multi_get returns the substates in the same order as queried, so ordered by transitions + for (data, substate) in data.transitions.iter().zip(substates) { + let update = match data.transition { StateTransitionType::Up => { - let value = substate.substate_value.map_or_else( + let value = include_values.then_some(substate.substate_value).flatten().map_or_else( || SubstateValueOrHash::Hash(substate.state_hash), |v| SubstateValueOrHash::Value(Box::new(v)), ); - SubstateUpdate::Create(SubstateCreatedProof { + SubstateUpdateProof::Create(SubstateCreatedProof { substate: SubstateData { substate_id: substate.substate_id, version: substate.version, @@ -1640,52 +1637,88 @@ impl<'tx, TAddr: NodeAddressable + Serialize + DeserializeOwned + 'tx> StateStor }, }) }, - StateTransitionType::Down => SubstateUpdate::Destroy(SubstateDestroyedProof { + StateTransitionType::Down => SubstateUpdateProof::Destroy(SubstateDestroyedProof { substate_id: substate.substate_id, version: substate.version, }), }; - transitions.push(StateTransition { - id: key, - state_version: value.state_version, - update, - }); - if transitions.len() == n { - break; - } + updates.push(update); } - Ok(transitions) - } - - fn state_transitions_get_last_id(&self, shard: Shard) -> Result { - // const OPERATION: &str = "state_transitions_get_last_id"; - let query = self.db().cf(state_transition::ByShardQuery)?; - let mut iter = query.query_prefix_range_key_iterator(Ordering::Descending, &shard); - - let key = iter.next().transpose()?.ok_or_else(|| StorageError::NotFound { - item: "StateTransition", - key: format!("last id in shard {}", shard), - })?; - - Ok(key) + Ok(StateVersionTransitions { + epoch: data.epoch, + shard, + state_version: version, + updates, + }) } - fn state_tree_nodes_get(&self, shard: Shard, key: &NodeKey) -> Result, StorageError> { + fn state_tree_nodes_get(&self, shard: Shard, key: &NodeKey) -> Result, StorageError> { const OPERATION: &str = "state_tree_nodes_get"; let cf = self.db().cf(StateTreeCfRef::default())?; let node = cf.get(&(shard, key), OPERATION)?; Ok(node) } + fn state_tree_nodes_get_all_by_state_version( + &self, + shard: Shard, + state_version: Version, + ) -> Result)>, StorageError> { + let cf = self.db().cf(state_tree::ByShardStateVersionQuery)?; + let iter = cf.query_prefix_range_iterator(Ordering::default(), &(shard, state_version)); + let nodes = iter + .map(|result| result.map(|((_, key), value)| (key, value))) + .collect::>()?; + Ok(nodes) + } + fn state_tree_versions_get_latest(&self, shard: Shard) -> Result, StorageError> { const OPERATION: &str = "state_tree_versions_get_latest"; - let query = self.db().cf(StateTreeShardVersionCf)?; - let version = query.get(&shard, OPERATION).optional()?; + let cf = self.db().cf(StateTreeShardVersionCf)?; + let version = cf.get(&shard, OPERATION).optional()?; Ok(version) } + fn state_tree_versions_get_latest_for_shard_group( + &self, + shard_group: ShardGroup, + ) -> Result { + const OPERATION: &str = "state_tree_versions_get_latest_for_shard_group"; + let mut shard_tree_versions = vec![0; shard_group.len() + 1]; + + let cf = self.db().cf(state_tree_shard_versions::ByShard)?; + let global = cf.get(&Shard::global(), OPERATION).optional()?.unwrap_or_default(); + shard_tree_versions[0] = global; + let sg_range = shard_group.start()..Shard::from(shard_group.end().as_u32() + 1); + let iter = cf.query_range_iterator(Ordering::Ascending, sg_range); + let mut shards_iter = shard_group.shard_iter(); + for result in iter { + let (shard, version) = result?; + + // Fill in the gaps with 0s for shard versions that are not yet set + for sg_shard in shards_iter.by_ref() { + if shard == sg_shard { + break; + } + + let index = + ShardStateVersions::shard_to_index(shard_group, sg_shard).expect("sg_shard must be in shard group"); + shard_tree_versions[index] = 0; + } + + let index = ShardStateVersions::shard_to_index(shard_group, shard) + .expect("BUG: we checked the end of the shard group, so shard must be in shard group"); + + shard_tree_versions[index] = version; + } + + let shard_tree_versions = + ShardStateVersions::from_vec(shard_tree_versions).expect("BUG: more shard tree versions than shards"); + Ok(shard_tree_versions) + } + fn epoch_checkpoint_get(&self, epoch: Epoch) -> Result { const OPERATION: &str = "epoch_checkpoint_get"; let cf = self.db().cf(EpochCheckpointCf)?; @@ -1693,13 +1726,6 @@ impl<'tx, TAddr: NodeAddressable + Serialize + DeserializeOwned + 'tx> StateStor Ok(checkpoint) } - fn previous_epoch_state_root_get(&self) -> Result { - const OPERATION: &str = "previous_epoch_state_root_get"; - let cf = self.db().cf(PreviousEpochStateRootCf)?; - let data = cf.get_by_default_key(OPERATION)?; - Ok(data) - } - fn foreign_substate_pledges_exists_for_transaction_and_address( &self, transaction_id: &TransactionId, diff --git a/crates/state_store_rocksdb/src/store.rs b/crates/state_store_rocksdb/src/store.rs index 5b8244c41c..72ae60d455 100644 --- a/crates/state_store_rocksdb/src/store.rs +++ b/crates/state_store_rocksdb/src/store.rs @@ -52,7 +52,6 @@ use crate::{ missing_transactions::MissingTransactionCf, parked_block::ParkedBlockCf, pending_state_tree_diff::PendingStateTreeDiffCf, - state_transition, state_transition::StateTransitionCf, state_tree::{StateTreeCf, StateTreeStaleNodesModel}, state_tree_shard_versions::StateTreeShardVersionCf, @@ -115,7 +114,6 @@ pub fn all_column_families_iter() -> impl Iterator { substate::HeadIndex::name(), substate::UnprunedDownedValuesIndex::name(), StateTransitionCf::name(), - state_transition::ShardSeqIndex::name(), ForeignSubstatePledgeCf::name(), PendingStateTreeDiffCf::name(), StateTreeCf::name(), diff --git a/crates/state_store_rocksdb/src/writer.rs b/crates/state_store_rocksdb/src/writer.rs index e2e2da8a1f..e9b0ed91d9 100644 --- a/crates/state_store_rocksdb/src/writer.rs +++ b/crates/state_store_rocksdb/src/writer.rs @@ -52,7 +52,6 @@ use tari_ootle_common_types::{ NumPreshards, ShardGroup, ToSubstateAddress, - VersionedSubstateId, }; use tari_ootle_storage::{ consensus_models::{ @@ -60,7 +59,6 @@ use tari_ootle_storage::{ BlockTransactionExecution, BurntUtxo, EpochCheckpoint, - EpochStateRoot, Evidence, ForeignParkedProposal, ForeignProposal, @@ -69,12 +67,14 @@ use tari_ootle_storage::{ LockConflict, NoVoteReason, PendingShardStateTreeDiff, - StateTransitionId, SubstateChange, + SubstateCreated, SubstateDestroyed, SubstateLock, SubstatePledges, SubstateRecord, + SubstateTransition, + SubstateUpdateBatch, TransactionPoolRecord, TransactionPoolStage, TransactionPoolStatusUpdate, @@ -87,7 +87,7 @@ use tari_ootle_storage::{ StateStoreWriteTransaction, StorageError, }; -use tari_state_tree::{Child, Nibble, Node, NodeKey, NodeType, StaleTreeNode, Version}; +use tari_state_tree::{Child, Nibble, Node, NodeKey, NodeType, StaleTreeNode, StateTreePayload, Version}; use tari_template_lib_types::crypto::RistrettoPublicKeyBytes; use tari_transaction::TransactionId; @@ -114,7 +114,6 @@ use crate::{ LastVotedCf, LeafBlockCf, LockedBlockCf, - PreviousEpochStateRootCf, }, burnt_utxo, burnt_utxo::BurntUtxoCf, @@ -138,8 +137,12 @@ use crate::{ parked_block::{ParkedBlockCf, ParkedBlockDataRef, ParkedBlockModelRef}, pending_state_tree_diff, pending_state_tree_diff::PendingStateTreeDiffCf, - state_transition, - state_transition::{StateTransitionCf, StateTransitionModelData, StateTransitionType}, + state_transition::{ + StateTransitionCf, + StateTransitionModelDataV1, + StateTransitionRecordData, + StateTransitionType, + }, state_tree, state_tree::{StateTreeCf, StateTreeStaleNodesModel}, state_tree_shard_versions::StateTreeShardVersionCf, @@ -1191,108 +1194,74 @@ impl<'tx, TAddr: NodeAddressable + 'tx> StateStoreWriteTransaction for RocksDbSt Ok(()) } - fn substates_create(&mut self, substate: &SubstateRecord) -> Result<(), StorageError> { - const OPERATION: &str = "substates_create"; - if substate.is_destroyed() { - return Err(StorageError::QueryError { - reason: format!( - "{OPERATION} calling substates_create with a destroyed SubstateRecord is not valid. substate_id = \ - {}", - substate.substate_id - ), - }); - } + fn substates_commit_batch(&mut self, update_batch: SubstateUpdateBatch) -> Result<(), StorageError> { + const OPERATION: &str = "substates_commit_batch"; let db = self.db(); - let address = substate.to_substate_address(); - db.cf(SubstateCf)?.put(&address, substate, OPERATION)?; - db.cf(substate::HeadIndex)?.put( - &substate.substate_id, - &SubstateHeadData { - version: substate.version(), - is_up: true, - }, - OPERATION, - )?; - - let shard_state_version = db - .cf(StateTreeShardVersionCf)? - .get(&substate.created_by_shard, OPERATION) - .optional()? - .unwrap_or_default(); - - let seq_index = db.cf(state_transition::ShardSeqIndex)?; - let seq = seq_index.get(&substate.created_by_shard, OPERATION).optional()?; - let next_seq = seq.map(|s| s + 1).unwrap_or(1); - - let id = StateTransitionId::new(substate.created_at_epoch, substate.created_by_shard, next_seq); - let transition = StateTransitionModelData { - substate_address: address, - state_version: shard_state_version, - transition: StateTransitionType::Up, - }; - - db.cf(StateTransitionCf)?.put(&id, &transition, OPERATION)?; - - seq_index.put(&substate.created_by_shard, &next_seq, OPERATION)?; - - Ok(()) - } - - fn substates_down( - &mut self, - versioned_substate_id: VersionedSubstateId, - shard: Shard, - epoch: Epoch, - destroyed_block_height: NodeHeight, - destroyed_qc_id: &QcId, - ) -> Result<(), StorageError> { - const OPERATION: &str = "substates_down"; - - let db = self.db(); let cf = db.cf(SubstateCf)?; + let head_cf = db.cf(substate::HeadIndex)?; + + for (shard, update) in update_batch.updates { + let mut transitions = Vec::with_capacity(update.transitions.len()); + + for transition in update.transitions { + match transition { + SubstateTransition::Up { + id, + version, + substate_or_hash, + } => { + let rec = SubstateRecord::new(id, version, substate_or_hash, SubstateCreated { + at_epoch: update_batch.epoch, + in_shard: shard, + at_state_version: update.state_version, + }); + + let address = rec.to_substate_address(); + cf.put(&address, &rec, OPERATION)?; + head_cf.put(rec.substate_id(), &SubstateHeadData { version, is_up: true }, OPERATION)?; + + transitions.push(StateTransitionRecordData { + substate_address: address, + transition: StateTransitionType::Up, + }); + }, + SubstateTransition::Down { id } => { + let address = id.to_substate_address(); + + let mut substate = cf.get(&address, OPERATION)?; + substate.destroyed = Some(SubstateDestroyed { + at_epoch: update_batch.epoch, + at_state_version: update.state_version, + }); + cf.put(&address, &substate, OPERATION)?; + db.cf(substate::HeadIndex)?.put( + &substate.substate_id, + &SubstateHeadData { + version: substate.version(), + is_up: false, + }, + OPERATION, + )?; + + transitions.push(StateTransitionRecordData { + substate_address: address, + transition: StateTransitionType::Down, + }); + }, + } + } - let address = versioned_substate_id.to_substate_address(); - let mut substate = cf.get(&address, OPERATION)?; - substate.destroyed = Some(SubstateDestroyed { - justify: *destroyed_qc_id, - by_block: destroyed_block_height, - at_epoch: epoch, - by_shard: shard, - }); - cf.put(&address, &substate, OPERATION)?; - db.cf(substate::HeadIndex)?.put( - &substate.substate_id, - &SubstateHeadData { - version: substate.version(), - is_up: false, - }, - OPERATION, - )?; - - let seq_index = db.cf(state_transition::ShardSeqIndex)?; - let seq = seq_index.get(&substate.created_by_shard, OPERATION).optional()?; - let next_seq = seq.map(|s| s + 1).unwrap_or(1); - - let transitions_cf = db.cf(StateTransitionCf)?; - - let shard_state_version = db - .cf(StateTreeShardVersionCf)? - .get(&shard, OPERATION) - .optional()? - .unwrap_or_default(); + let transition = StateTransitionModelDataV1 { + epoch: update_batch.epoch, + state_version: update.state_version, + transitions, + }; - let data = StateTransitionModelData { - substate_address: address, - state_version: shard_state_version, - transition: StateTransitionType::Down, - }; - let id = StateTransitionId::new(epoch, shard, next_seq); - transitions_cf.put(&id, &data, OPERATION)?; - let unpruned_cf = db.cf(substate::UnprunedDownedValuesIndex)?; - unpruned_cf.put(&(id.epoch(), id.shard(), id.seq()), &address, OPERATION)?; - seq_index.put(&shard, &next_seq, OPERATION)?; + db.cf(StateTransitionCf)? + .put(&(shard, update.state_version), &transition, OPERATION)?; + } Ok(()) } @@ -1416,7 +1385,7 @@ impl<'tx, TAddr: NodeAddressable + 'tx> StateStoreWriteTransaction for RocksDbSt fn state_tree_nodes_batch_insert( &mut self, shard: Shard, - nodes: Vec<(NodeKey, Node)>, + nodes: Vec<(NodeKey, Node)>, ) -> Result<(), StorageError> { const OPERATION: &str = "state_tree_nodes_insert"; let cf = self.db().cf(StateTreeCf)?; @@ -1554,14 +1523,6 @@ impl<'tx, TAddr: NodeAddressable + 'tx> StateStoreWriteTransaction for RocksDbSt Ok(()) } - fn previous_epoch_state_root_set(&mut self, epoch_state_root: &EpochStateRoot) -> Result<(), StorageError> { - const OPERATION: &str = "epoch_state_root_set"; - self.db() - .cf(PreviousEpochStateRootCf)? - .put(&ByteColumn, epoch_state_root, OPERATION)?; - Ok(()) - } - fn burnt_utxos_insert(&mut self, burnt_utxo: &BurntUtxo) -> Result<(), StorageError> { const OPERATION: &str = "burnt_utxos_insert"; diff --git a/crates/state_store_tests/src/block_diffs.rs b/crates/state_store_tests/src/block_diffs.rs index ccfb3c078d..37e487811a 100644 --- a/crates/state_store_tests/src/block_diffs.rs +++ b/crates/state_store_tests/src/block_diffs.rs @@ -38,7 +38,7 @@ fn block_diffs_operations(db: impl StateStore) { let block_id9 = *block9.id(); let substate_id = create_random_substate_id(); let version = 0; - let substate_record = build_substate_record(&substate_id, version); + let substate_record = build_substate_record(&substate_id, version, 1); let change = SubstateChange::Up { id: substate_id.clone(), shard: block9.shard_group().start(), @@ -64,7 +64,7 @@ fn block_diffs_operations(db: impl StateStore) { // block_diffs_get let res = tx.block_diffs_get(&block_id9).unwrap(); - assert_eq!(res.changes.len(), 2); + assert_eq!(res.changes().len(), 2); let change = tx .block_diffs_get_last_change_for_substate(&block_id9, &substate_id) @@ -82,7 +82,7 @@ fn block_diffs_operations(db: impl StateStore) { // block_diffs_remove tx.block_diffs_remove(&block_id9).unwrap(); let res = tx.block_diffs_get(&block_id9).unwrap(); - assert_eq!(res.changes.len(), 0); + assert_eq!(res.changes().len(), 0); tx.rollback().unwrap(); } diff --git a/crates/state_store_tests/src/blocks.rs b/crates/state_store_tests/src/blocks.rs index 4d261e2dd6..618a942747 100644 --- a/crates/state_store_tests/src/blocks.rs +++ b/crates/state_store_tests/src/blocks.rs @@ -107,6 +107,7 @@ mod block_parent_operations { let zero_block = Block::zero_block(network, NumPreshards::P64); zero_block.insert(&mut tx).unwrap(); + let shard_group = ShardGroup::all_shards(NumPreshards::P64); let block1 = Block::create( network, *zero_block.id(), @@ -114,7 +115,7 @@ mod block_parent_operations { None, NodeHeight(1), Epoch(0), - ShardGroup::all_shards(NumPreshards::P64), + shard_group, Default::default(), // Need to have a command in, otherwise this block will not be included internally in the query because it // cannot cause a state change without any commands @@ -136,7 +137,7 @@ mod block_parent_operations { None, NodeHeight(1), Epoch(0), - ShardGroup::all_shards(NumPreshards::P64), + shard_group, Default::default(), // Need to have a command in, otherwise this block will not be included internally in the query because it // cannot cause a state change without any commands @@ -237,6 +238,7 @@ mod block_query_operations { tx.blocks_set_qcs(zero_block.id(), Some(&QcId::zero()), Some(&QcId::zero())) .unwrap(); + let shard_group = ShardGroup::all_shards(NumPreshards::P64); let block1 = Block::create( network, *zero_block.id(), @@ -244,7 +246,7 @@ mod block_query_operations { None, NodeHeight(1), Epoch(0), - ShardGroup::all_shards(NumPreshards::P64), + shard_group, Default::default(), // Need to have a command in, otherwise this block will not be included internally in the query because it // cannot cause a state change without any commands @@ -269,7 +271,7 @@ mod block_query_operations { None, NodeHeight(2), Epoch(0), - ShardGroup::all_shards(NumPreshards::P64), + shard_group, Default::default(), // Need to have a command in, otherwise this block will not be included internally in the query because it // cannot cause a state change without any commands @@ -301,7 +303,7 @@ mod block_query_operations { // Height 2 to test forks NodeHeight(2), Epoch(0), - ShardGroup::all_shards(NumPreshards::P64), + shard_group, Default::default(), // Need to have a command in, otherwise this block will not be included internally in the query because it // cannot cause a state change without any commands diff --git a/crates/state_store_tests/src/foreign_proposals.rs b/crates/state_store_tests/src/foreign_proposals.rs index 3ec8a3587d..b85ed38e18 100644 --- a/crates/state_store_tests/src/foreign_proposals.rs +++ b/crates/state_store_tests/src/foreign_proposals.rs @@ -33,6 +33,8 @@ fn foreign_proposals_operations(db: impl StateStore) { let proposal1 = create_foreign_proposal(*zero_block.id(), EPOCH); tx.foreign_proposals_save(&proposal1).unwrap(); + let shard_group = ShardGroup::all_shards(NumPreshards::P64); + let block1 = Block::create( network, *zero_block.id(), @@ -40,7 +42,7 @@ fn foreign_proposals_operations(db: impl StateStore) { None, NodeHeight(2), EPOCH, - ShardGroup::all_shards(NumPreshards::P64), + shard_group, Default::default(), [Command::ForeignProposal(proposal1.to_atom())] .iter() @@ -65,7 +67,7 @@ fn foreign_proposals_operations(db: impl StateStore) { None, NodeHeight(2), EPOCH, - ShardGroup::all_shards(NumPreshards::P64), + shard_group, Default::default(), Default::default(), Default::default(), diff --git a/crates/state_store_tests/src/helpers.rs b/crates/state_store_tests/src/helpers.rs index b52f6782c1..d6e28e3383 100644 --- a/crates/state_store_tests/src/helpers.rs +++ b/crates/state_store_tests/src/helpers.rs @@ -28,9 +28,18 @@ use tari_common_types::types::FixedHash; use tari_consensus_types::{BlockId, Decision, LeafBlock, ProposalCertificate, QcId}; use tari_engine_types::{ component::{ComponentBody, ComponentHeader}, - substate::{hash_substate, Substate, SubstateId, SubstateValue}, + substate::{hash_substate, SubstateId, SubstateValue}, +}; +use tari_ootle_common_types::{ + Epoch, + ExtraData, + Network, + NodeHeight, + NumPreshards, + ShardGroup, + VersionedSubstateId, + VersionedSubstateIdRef, }; -use tari_ootle_common_types::{shard::Shard, Epoch, ExtraData, Network, NodeHeight, NumPreshards, ShardGroup}; use tari_ootle_storage::{ consensus_models::{ Block, @@ -40,7 +49,9 @@ use tari_ootle_storage::{ CommandsCommitProof, ForeignProposal, ForeignProposalRecord, + SubstateCreated, SubstateRecord, + SubstateUpdateBatch, TransactionAtom, }, StateStoreReadTransaction, @@ -48,6 +59,7 @@ use tari_ootle_storage::{ }; use tari_sidechain::{CommitProofElement, QuorumDecision, SidechainBlockCommitProof, SidechainBlockHeader}; use tari_state_store_rocksdb::{DatabaseOptions, RocksDbStateStore}; +use tari_state_tree::Version; use tari_template_lib::{ auth::OwnerRule, models::ComponentAddress, @@ -113,7 +125,7 @@ pub fn transaction_id_from_seed(seed: u32) -> TransactionId { TransactionId::new(buf) } -pub fn build_substate_record(substate_id: &SubstateId, version: u32) -> SubstateRecord { +pub fn build_substate_record(substate_id: &SubstateId, version: u32, state_version: Version) -> SubstateRecord { let entity_id = substate_id.to_object_key().as_entity_id(); let value = build_substate_value(Some(entity_id)); SubstateRecord { @@ -121,10 +133,11 @@ pub fn build_substate_record(substate_id: &SubstateId, version: u32) -> Substate version, state_hash: hash_substate(&value, version), substate_value: Some(value), - created_justify: QcId::zero(), - created_block: BlockId::zero(), - created_by_shard: Shard::first(), - created_at_epoch: Epoch::zero(), + created: SubstateCreated { + at_epoch: Epoch::zero(), + in_shard: VersionedSubstateIdRef::new(substate_id, version).to_shard(TEST_NUM_PRESHARDS), + at_state_version: state_version, + }, destroyed: None, } } @@ -149,6 +162,35 @@ pub fn build_substate_value(entity_id: Option) -> SubstateValue { }) } +pub fn create_substate_update_batch<'a, I: IntoIterator>( + epoch: Epoch, + substates: I, +) -> SubstateUpdateBatch { + let mut batch = SubstateUpdateBatch::new(epoch); + for substate in substates { + if let Some(destroyed) = &substate.destroyed { + batch.add_transition( + substate.to_versioned_substate_id().to_shard(TEST_NUM_PRESHARDS), + destroyed.at_state_version, + tari_ootle_storage::consensus_models::SubstateTransition::Down { + id: VersionedSubstateId::new(substate.substate_id.clone(), substate.version), + }, + ); + } else { + batch.add_transition( + substate.to_versioned_substate_id().to_shard(TEST_NUM_PRESHARDS), + substate.created().at_state_version, + tari_ootle_storage::consensus_models::SubstateTransition::Up { + id: substate.substate_id.clone(), + version: substate.version, + substate_or_hash: substate.clone().into_substate_value_or_hash(), + }, + ); + } + } + batch +} + pub fn substate_id_tx_seed(transaction_id: TransactionId, seed: u32) -> SubstateId { let mut buf = [0u8; EntityId::LENGTH]; buf[..].copy_from_slice(&transaction_id.as_hash().as_slice()[..EntityId::LENGTH]); @@ -188,19 +230,28 @@ pub fn substate_value_for_entity(entity_id: EntityId) -> SubstateValue { } pub fn gen_substates( + epoch: Epoch, + state_version: Version, range: impl IntoIterator, version: u32, -) -> impl Iterator { +) -> impl Iterator { range.into_iter().map(move |i| { let substate_id = substate_id_seed(i); let value = substate_value_for_entity(substate_id.to_object_key().as_entity_id()); - (substate_id, Substate::new(version, value)) + let shard = VersionedSubstateIdRef::new(&substate_id, version).to_shard(TEST_NUM_PRESHARDS); + SubstateRecord::new(substate_id, version, value, SubstateCreated { + at_epoch: epoch, + in_shard: shard, + at_state_version: state_version, + }) }) } +// track_caller allows a panic to include the caller's location in the error message +#[track_caller] pub fn assert_eq_debug(a: &T, b: &T) where T: std::fmt::Debug { - assert_eq!(format!("{:?}", a), format!("{:?}", b),); + assert_eq!(format!("{:?}", a), format!("{:?}", b)); } pub fn create_random_block_id() -> BlockId { @@ -223,6 +274,7 @@ pub fn create_block(parent: Option<&Block>) -> Block { // This prevents all blocks to have the same hash/id let random_merkle_root = create_random_hash(); + let shard_group = ShardGroup::all_shards(num_preshards()); Block::create( network, @@ -231,7 +283,7 @@ pub fn create_block(parent: Option<&Block>) -> Block { None, NodeHeight(1), Epoch(0), - ShardGroup::all_shards(num_preshards()), + shard_group, Default::default(), // Need to have a command in, otherwise this block will not be included internally in the query because it // cannot cause a state change without any commands @@ -255,6 +307,7 @@ pub fn create_block_with_qc(parent: &LeafBlock) -> Block { let random_merkle_root = create_random_hash(); let qc = create_qc(parent); + let shard_group = parent.shard_group(); Block::create( network, @@ -263,7 +316,7 @@ pub fn create_block_with_qc(parent: &LeafBlock) -> Block { None, parent.height() + NodeHeight(1), parent.epoch(), - ShardGroup::all_shards(num_preshards()), + shard_group, Default::default(), // Need to have a command in, otherwise this block will not be included internally in the query because it // cannot cause a state change without any commands diff --git a/crates/state_store_tests/src/misc.rs b/crates/state_store_tests/src/misc.rs index c317244cc1..b16fbf2824 100644 --- a/crates/state_store_tests/src/misc.rs +++ b/crates/state_store_tests/src/misc.rs @@ -16,7 +16,7 @@ use tari_consensus_types::{ }; use tari_ootle_common_types::{optional::Optional, Epoch, Network, NodeHeight, ShardGroup}; use tari_ootle_storage::{ - consensus_models::{Block, EndOfEpochCommand, EpochCheckpoint}, + consensus_models::{Block, EndOfEpochCommand, EpochCheckpoint, TreeRootSummary}, StateStore, StateStoreReadTransaction, StateStoreWriteTransaction, @@ -166,8 +166,11 @@ fn miscellaneous_operations(db: impl StateStore) { // epoch checkpoints let shard_group = ShardGroup::all_shards(TEST_NUM_PRESHARDS); let block = Block::zero_block(Network::LocalNet, TEST_NUM_PRESHARDS); - let mut shard_roots = IndexMap::new(); - shard_roots.insert(shard_group.start(), TreeHash::zero()); + let mut shard_summary = IndexMap::new(); + shard_summary.insert(shard_group.start(), TreeRootSummary { + root_hash: TreeHash::zero(), + state_version: 0, + }); let key = TreeHash::new([1; 32]); let (_, inclusion_proof) = compute_proof_for_hashes([key].into_iter(), key).unwrap(); let commit_proof = SidechainBlockCommitProof { @@ -190,7 +193,7 @@ fn miscellaneous_operations(db: impl StateStore) { proof_elements: vec![], }; let proof = CommandCommitProof::new(EndOfEpochCommand, commit_proof, inclusion_proof); - let epoch_checkpoint = EpochCheckpoint::new(proof, shard_roots); + let epoch_checkpoint = EpochCheckpoint::new(proof, shard_summary); tx.epoch_checkpoint_save(&epoch_checkpoint).unwrap(); let res = tx.epoch_checkpoint_get(block.epoch()).unwrap(); diff --git a/crates/state_store_tests/src/missing_transactions.rs b/crates/state_store_tests/src/missing_transactions.rs index c7c7901619..66157fdd58 100644 --- a/crates/state_store_tests/src/missing_transactions.rs +++ b/crates/state_store_tests/src/missing_transactions.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: BSD-3-Clause use tari_common_types::types::FixedHash; -use tari_ootle_common_types::{Epoch, ExtraData, Network, NodeHeight, NumPreshards, ShardGroup}; +use tari_ootle_common_types::{Epoch, ExtraData, Network, NodeHeight}; use tari_ootle_storage::{ consensus_models::{Block, Command}, StateStore, @@ -36,7 +36,7 @@ fn missing_transactions_operations(db: impl StateStore) { None, NodeHeight(1), Epoch(0), - ShardGroup::all_shards(NumPreshards::P64), + genesis.shard_group(), Default::default(), // Need to have a command in, otherwise this block will not be included internally in the query because it // cannot cause a state change without any commands diff --git a/crates/state_store_tests/src/state_transitions.rs b/crates/state_store_tests/src/state_transitions.rs index 606aefee5b..a76680738e 100644 --- a/crates/state_store_tests/src/state_transitions.rs +++ b/crates/state_store_tests/src/state_transitions.rs @@ -1,17 +1,14 @@ // Copyright 2025 The Tari Project // SPDX-License-Identifier: BSD-3-Clause -use tari_consensus_types::{BlockId, LeafBlock}; -use tari_ootle_common_types::{shard::Shard, Epoch, Network, NodeHeight, ShardGroup}; -use tari_ootle_storage::{ - consensus_models::{Block, StateTransitionId, SubstateRecord}, - StateStore, - StateStoreReadTransaction, - StateStoreWriteTransaction, -}; +use std::collections::{HashMap, HashSet}; + +use tari_ootle_common_types::{Epoch, Network}; +use tari_ootle_storage::{consensus_models::Block, StateStore, StateStoreReadTransaction, StateStoreWriteTransaction}; +use tari_state_tree::Version; use crate::{ - helpers::{create_block_with_qc, create_rocksdb, gen_substates}, + helpers::{create_rocksdb, create_substate_update_batch, gen_substates}, TEST_NUM_PRESHARDS, }; @@ -23,94 +20,54 @@ fn rocksdb() { fn operations(db: impl StateStore) { let num_transitions = 100; // Makes double - const SHARD: Shard = Shard::first(); + const EPOCH: Epoch = Epoch::zero(); let mut tx = db.create_write_tx().unwrap(); let zero_block = Block::zero_block(Network::LocalNet, TEST_NUM_PRESHARDS); zero_block.insert(&mut tx).unwrap(); - let substates = gen_substates(0..num_transitions, 0); - let dummy_parent = LeafBlock { - block_id: BlockId::zero(), - height: NodeHeight(0), - epoch: Epoch(0), - shard_group: ShardGroup::all_shards(TEST_NUM_PRESHARDS), - }; - for (key, value) in substates { - let block = create_block_with_qc(&dummy_parent); - tx.substates_create(&SubstateRecord::new( - key, - value.version(), - value.into_substate_value(), - SHARD, - Epoch(0), - *zero_block.id(), - block.justify().calculate_id(), - )) - .unwrap(); - } + let mut shards = HashMap::new(); + let substates = gen_substates(EPOCH, 1, 0..num_transitions, 0).collect::>(); + shards.insert( + 1 as Version, + ( + substates.len(), + substates.iter().map(|s| s.shard()).collect::>(), + ), + ); + let batch = create_substate_update_batch(Epoch::zero(), &substates); + tx.substates_commit_batch(batch).unwrap(); // Add a couple for a different shard - let substates = gen_substates(num_transitions..num_transitions + 2, 0); - for (key, value) in substates { - let block = create_block_with_qc(&dummy_parent); - tx.substates_create(&SubstateRecord::new( - key, - value.version(), - value.into_substate_value(), - Shard::from(2), - Epoch(0), - *zero_block.id(), - block.justify().calculate_id(), - )) - .unwrap(); - } - - let substates = gen_substates(0..num_transitions, 1); - let dummy_parent = LeafBlock { - block_id: BlockId::zero(), - height: NodeHeight(10000), - epoch: Epoch(1000), - shard_group: ShardGroup::all_shards(TEST_NUM_PRESHARDS), - }; - for (key, value) in substates { - let block = create_block_with_qc(&dummy_parent); - tx.substates_create(&SubstateRecord::new( - key, - value.version(), - value.into_substate_value(), - SHARD, - Epoch(1000), - *zero_block.id(), - block.justify().calculate_id(), - )) - .unwrap(); - } - - let last_id = tx.state_transitions_get_last_id(SHARD).unwrap(); - assert_eq!(last_id.shard(), SHARD); - assert_eq!(last_id.seq(), u64::from(num_transitions) * 2); - assert_eq!(last_id.epoch(), Epoch(1000)); - - let transitions = tx.state_transitions_get_n_after(10000, last_id, Epoch(1000)).unwrap(); - assert_eq!(transitions.len(), 0); + let substates = gen_substates(EPOCH, 2, num_transitions..num_transitions + 2, 0).collect::>(); + shards.insert( + 2, + ( + substates.len(), + substates.iter().map(|s| s.shard()).collect::>(), + ), + ); + let batch = create_substate_update_batch(Epoch::zero(), &substates); + tx.substates_commit_batch(batch).unwrap(); - let prev_id = StateTransitionId::new(Epoch(0), SHARD, 0); - let transitions = tx.state_transitions_get_n_after(10000, prev_id, Epoch(1000)).unwrap(); - assert_eq!(transitions.len(), 100); + let substates = gen_substates(EPOCH, 3, 0..num_transitions, 1).collect::>(); + shards.insert( + 3, + ( + substates.len(), + substates.iter().map(|s| s.shard()).collect::>(), + ), + ); + let batch = create_substate_update_batch(Epoch::zero(), &substates); + tx.substates_commit_batch(batch).unwrap(); - let prev_id = StateTransitionId::new(Epoch(1000), SHARD, last_id.seq() - 10); - let transitions = tx.state_transitions_get_n_after(10000, prev_id, Epoch(1001)).unwrap(); - for (i, transition) in transitions.iter().enumerate() { - assert_eq!(transition.id.shard(), SHARD); - assert_eq!(transition.id.epoch(), Epoch(1000)); - assert_eq!(transition.id.seq(), last_id.seq() - 10 + i as u64 + 1); + for (state_version, (num_substates, shards)) in &shards { + for shard in shards { + let transitions = tx.state_transitions_get_after(*shard, *state_version, false).unwrap(); + assert_eq!(transitions.epoch, EPOCH); + assert_eq!(transitions.state_version, *state_version); + assert_eq!(transitions.shard, *shard); + assert_eq!(transitions.updates.len(), *num_substates); + } } - assert_eq!(transitions.len(), 10); - - let prev_seq = num_transitions - 10; - let id = StateTransitionId::new(Epoch(0), SHARD, u64::from(prev_seq)); - - let transitions = tx.state_transitions_get_n_after(10000, id, Epoch(100)).unwrap(); - assert_eq!(transitions.len(), 10); } diff --git a/crates/state_store_tests/src/state_tree.rs b/crates/state_store_tests/src/state_tree.rs index 5590e5c240..b2e5e82227 100644 --- a/crates/state_store_tests/src/state_tree.rs +++ b/crates/state_store_tests/src/state_tree.rs @@ -3,7 +3,7 @@ use tari_ootle_common_types::{optional::Optional, shard::Shard, ShardGroup}; use tari_ootle_storage::{StateStore, StateStoreReadTransaction, StateStoreWriteTransaction, StorageError}; -use tari_state_tree::{NibblePath, Node, NodeKey, StaleTreeNode, Version}; +use tari_state_tree::{NibblePath, Node, NodeKey, StaleTreeNode, StateTreePayload}; use crate::{ helpers::{assert_eq_debug, create_rocksdb}, @@ -78,7 +78,7 @@ fn state_tree_operations(db: impl StateStore, num_nodes: usize) { .unwrap(); } -fn gen_nodes(version: u64, num: usize) -> impl Iterator)> { +fn gen_nodes(version: u64, num: usize) -> impl Iterator)> { (0..num as u64).map(move |i| { let node = Node::Null; // No possibility of key collisions diff --git a/crates/state_store_tests/src/substates.rs b/crates/state_store_tests/src/substates.rs index ce6ac825f3..bb383c5db8 100644 --- a/crates/state_store_tests/src/substates.rs +++ b/crates/state_store_tests/src/substates.rs @@ -3,14 +3,18 @@ use std::collections::HashSet; -use tari_consensus_types::QcId; use tari_engine_types::substate::SubstateId; -use tari_ootle_common_types::{shard::Shard, Epoch, Network, NodeHeight, VersionedSubstateId, VersionedSubstateIdRef}; -use tari_ootle_storage::{consensus_models::Block, StateStore, StateStoreReadTransaction, StateStoreWriteTransaction}; +use tari_ootle_common_types::{Epoch, Network, VersionedSubstateId, VersionedSubstateIdRef}; +use tari_ootle_storage::{ + consensus_models::{Block, SubstateUpdateBatch}, + StateStore, + StateStoreReadTransaction, + StateStoreWriteTransaction, +}; use tari_template_lib::{models::ComponentAddress, types::ObjectKey}; use crate::{ - helpers::{assert_eq_debug, build_substate_record, create_rocksdb}, + helpers::{assert_eq_debug, build_substate_record, create_rocksdb, create_substate_update_batch}, TEST_NUM_PRESHARDS, }; @@ -33,30 +37,18 @@ fn operations(db: impl StateStore) { // substate 1 let substate1_id = substate_id(1); - let mut substate1 = build_substate_record(&substate1_id, 0); - substate1.created_block = *zero_block.id(); + let substate1 = build_substate_record(&substate1_id, 0, 1); let substate1_address = substate1.to_substate_address(); - tx.substates_create(&substate1).unwrap(); - tx.substates_down( - VersionedSubstateId::new(substate1_id.clone(), 0), - Shard::first(), - Epoch(123), - NodeHeight(123), - &QcId::zero(), - ) - .unwrap(); // substate 1 (version 1) - let mut substate1b = build_substate_record(&substate1_id, 1); - substate1b.created_block = *zero_block.id(); + let substate1b = build_substate_record(&substate1_id, 1, 1); let substate1b_address = substate1b.to_substate_address(); - tx.substates_create(&substate1b).unwrap(); - // substate 2 let substate2_id = substate_id(2); - let mut substate2 = build_substate_record(&substate2_id, 0); - substate2.created_block = *zero_block.id(); + let substate2 = build_substate_record(&substate2_id, 0, 1); let substate2_address = substate2.to_substate_address(); - tx.substates_create(&substate2).unwrap(); + + let batch = create_substate_update_batch(Epoch::zero(), [&substate1, &substate1b, &substate2]); + tx.substates_commit_batch(batch).unwrap(); // check that we can get all the newly inserted substates let res = tx.substates_get(&substate1_address).unwrap(); @@ -137,19 +129,18 @@ fn operations(db: impl StateStore) { assert!(res.destroyed.is_none()); let versioned_substate_id = VersionedSubstateId::new(substate2.substate_id, substate2.version); - let shard = Shard::first(); + let shard = versioned_substate_id.to_shard(TEST_NUM_PRESHARDS); let epoch = Epoch::zero(); - let destroyed_block_height = NodeHeight::zero(); - let destroyed_qc_id = QcId::zero(); - tx.substates_down( - versioned_substate_id, + let mut batch = SubstateUpdateBatch::new(epoch); + batch.add_transition( shard, - epoch, - destroyed_block_height, - &destroyed_qc_id, - ) - .unwrap(); + 2, + tari_ootle_storage::consensus_models::SubstateTransition::Down { + id: versioned_substate_id.clone(), + }, + ); + tx.substates_commit_batch(batch).unwrap(); let res = tx.substates_get(&substate2_address).unwrap(); assert!(res.destroyed.is_some()); diff --git a/crates/state_store_tests/src/transactions.rs b/crates/state_store_tests/src/transactions.rs index 837505a0c0..b55fdd4bcf 100644 --- a/crates/state_store_tests/src/transactions.rs +++ b/crates/state_store_tests/src/transactions.rs @@ -10,7 +10,7 @@ use tari_engine_types::{ fees::{FeeBreakdown, FeeReceipt}, substate::SubstateDiff, }; -use tari_ootle_common_types::{Epoch, ExtraData, NodeHeight, ShardGroup, SubstateRequirement}; +use tari_ootle_common_types::{Epoch, ExtraData, NodeHeight, SubstateRequirement}; use tari_ootle_storage::{ consensus_models::{ Block, @@ -60,7 +60,7 @@ mod confirm_all_transitions { tx.blocks_set_qcs(zero_block.id(), Some(&QcId::zero()), Some(&QcId::zero())) .unwrap(); - let shard_group = ShardGroup::all_shards(TEST_NUM_PRESHARDS); + let shard_group = zero_block.shard_group(); let block1 = Block::create( network, diff --git a/crates/state_tree/src/lib.rs b/crates/state_tree/src/lib.rs index 354e0c278c..0488e251e4 100644 --- a/crates/state_tree/src/lib.rs +++ b/crates/state_tree/src/lib.rs @@ -18,3 +18,6 @@ pub use traits::*; mod tree; pub use tree::*; + +/// The payload type used in the state tree. This is a reference to a particular substate (i.e. SubstateAddress). +pub type StateTreePayload = tari_ootle_common_types::SubstateAddress; diff --git a/crates/state_tree/src/tree.rs b/crates/state_tree/src/tree.rs index 580a98fa92..f346e6567a 100644 --- a/crates/state_tree/src/tree.rs +++ b/crates/state_tree/src/tree.rs @@ -19,12 +19,13 @@ use tari_jellyfish::{ TreeUpdateBatch, Version, }; -use tari_ootle_common_types::VersionedSubstateId; +use tari_ootle_common_types::{ToSubstateAddress, VersionedSubstateId}; use crate::{ error::StateTreeError, key_mapper::{DbKeyMapper, HashIdentityKeyMapper, SpreadPrefixKeyMapper}, memory_store::MemoryTreeStore, + StateTreePayload, TreeStoreBatchWriter, SPARSE_MERKLE_PLACEHOLDER_HASH, }; @@ -48,12 +49,12 @@ impl<'a, S, M> StateTree<'a, S, M> { } } -impl, M: DbKeyMapper> StateTree<'_, S, M> { +impl, M: DbKeyMapper> StateTree<'_, S, M> { pub fn get_proof( &self, version: Version, key: &VersionedSubstateId, - ) -> Result<(LeafKey, Option>, SparseMerkleProofExt), StateTreeError> { + ) -> Result<(LeafKey, Option>, SparseMerkleProofExt), StateTreeError> { let jmt = JellyfishMerkleTree::new(self.store); let key = M::map_to_leaf_key(key); let (maybe_value, proof) = jmt.get_with_proof_ext(key.as_ref(), version)?; @@ -71,14 +72,14 @@ impl, M: DbKeyMapper> StateTree current_version: Option, next_version: Version, changes: I, - ) -> Result<(TreeHash, StateHashTreeDiff), StateTreeError> { + ) -> Result<(TreeHash, StateHashTreeDiff), StateTreeError> { let (root_hash, update_batch) = calculate_substate_changes::<_, M, _>(self.store, current_version, next_version, changes)?; Ok((root_hash, update_batch.into())) } } -impl, M: DbKeyMapper> StateTree<'_, S, M> { +impl, M: DbKeyMapper> StateTree<'_, S, M> { /// Stores the substate changes in the state tree and returns the new root hash. pub fn put_substate_changes>( &mut self, @@ -91,7 +92,7 @@ impl, M: DbKeyMapper> StateTree<'_, S Ok(root_hash) } - fn commit_diff(&mut self, diff: StateHashTreeDiff) -> Result<(), StateTreeError> { + fn commit_diff(&mut self, diff: StateHashTreeDiff) -> Result<(), StateTreeError> { for (key, node) in diff.new_nodes { log::debug!("Inserting node: {}", key); self.store.insert_node(key, node)?; @@ -106,8 +107,10 @@ impl, M: DbKeyMapper> StateTree<'_, S } } -impl + TreeStoreBatchWriter, M: DbKeyMapper> - StateTree<'_, S, M> +impl< + S: TreeStoreReader + TreeStoreBatchWriter, + M: DbKeyMapper, + > StateTree<'_, S, M> { /// Stores the substate changes in the state tree and returns the new root hash. pub fn batch_put_substate_changes>( @@ -171,26 +174,26 @@ impl, M: DbKeyMapper> StateTree<'_, S, M> { /// Calculates the new root hash and tree updates for the given substate changes. fn calculate_substate_changes< - S: TreeStoreReader, + S: TreeStoreReader, M: DbKeyMapper, I: IntoIterator, >( store: &mut S, current_version: Option, - next_version: Version, + version: Version, changes: I, -) -> Result<(TreeHash, TreeUpdateBatch), StateTreeError> { +) -> Result<(TreeHash, TreeUpdateBatch), StateTreeError> { let jmt = JellyfishMerkleTree::new(store); let changes = changes.into_iter().map(|ch| match ch { SubstateTreeChange::Up { id, value_hash } => ( M::map_to_leaf_key(&id), - Some((TreeHash::new(value_hash.into_array()), next_version)), + Some((TreeHash::new(value_hash.into_array()), id.to_substate_address())), ), SubstateTreeChange::Down { id } => (M::map_to_leaf_key(&id), None), }); - let (root_hash, update_result) = jmt.batch_put_value_set(changes, None, current_version, next_version)?; + let (root_hash, update_result) = jmt.batch_put_value_set(changes, None, current_version, version)?; Ok((root_hash, update_result)) } diff --git a/crates/state_tree/tests/support.rs b/crates/state_tree/tests/support.rs index 0755cd6c48..9dc23c8a56 100644 --- a/crates/state_tree/tests/support.rs +++ b/crates/state_tree/tests/support.rs @@ -5,7 +5,13 @@ use tari_common_types::types::FixedHash; use tari_engine_types::{hashing::substate_value_hasher32, substate::SubstateId}; use tari_jellyfish::{LeafKey, TreeHash, TreeStore, Version}; use tari_ootle_common_types::VersionedSubstateId; -use tari_state_tree::{key_mapper::DbKeyMapper, memory_store::MemoryTreeStore, StateTree, SubstateTreeChange}; +use tari_state_tree::{ + key_mapper::DbKeyMapper, + memory_store::MemoryTreeStore, + StateTree, + StateTreePayload, + SubstateTreeChange, +}; use tari_template_lib::{models::ComponentAddress, types::ObjectKey}; pub fn make_value(seed: u8) -> VersionedSubstateId { @@ -45,7 +51,7 @@ pub struct HashTreeTester { pub current_version: Option, } -impl> HashTreeTester { +impl> HashTreeTester { pub fn new(tree_store: S, current_version: Option) -> Self { Self { tree_store, @@ -79,7 +85,7 @@ impl> HashTreeTester { } } -impl HashTreeTester> { +impl HashTreeTester> { pub fn new_empty() -> Self { Self::new(MemoryTreeStore::new(), None) } diff --git a/crates/state_tree/tests/test.rs b/crates/state_tree/tests/test.rs index 59b9f06b1e..416542a29e 100644 --- a/crates/state_tree/tests/test.rs +++ b/crates/state_tree/tests/test.rs @@ -5,6 +5,7 @@ use std::collections::{BTreeSet, HashSet}; use tari_jellyfish::{StaleTreeNode, Version, SPARSE_MERKLE_PLACEHOLDER_HASH}; +use tari_ootle_common_types::ToSubstateAddress; use tari_state_tree::memory_store::MemoryTreeStore; use crate::support::{change, hash_value_from_seed, make_value, HashTreeTester}; @@ -191,15 +192,15 @@ fn proofs() { let tree = tester.create_state_tree(); let (key, proof_value, proof) = tree.get_proof(3, &make_value(1)).unwrap(); let hash = hash_value_from_seed(30); - assert_eq!(proof_value, Some((hash, 1, 1))); + assert_eq!(proof_value, Some((hash, make_value(1).to_substate_address(), 1))); proof.verify_inclusion(&root_hash, &key, &hash).unwrap(); let (key, proof_value, proof) = tree.get_proof(3, &make_value(2)).unwrap(); let hash = hash_value_from_seed(40); - assert_eq!(proof_value, Some((hash, 2, 2))); + assert_eq!(proof_value, Some((hash, make_value(2).to_substate_address(), 2))); proof.verify_inclusion(&root_hash, &key, &hash).unwrap(); let (key, proof_value, proof) = tree.get_proof(3, &make_value(3)).unwrap(); let hash = hash_value_from_seed(50); - assert_eq!(proof_value, Some((hash, 3, 3))); + assert_eq!(proof_value, Some((hash, make_value(3).to_substate_address(), 3))); proof.verify_inclusion(&root_hash, &key, &hash).unwrap(); let (key, proof_value, proof) = tree.get_proof(3, &make_value(3)).unwrap(); proof diff --git a/crates/storage/src/consensus_models/block.rs b/crates/storage/src/consensus_models/block.rs index 3a703b8b24..f5b8a0fb60 100644 --- a/crates/storage/src/consensus_models/block.rs +++ b/crates/storage/src/consensus_models/block.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: BSD-3-Clause use std::{ - collections::BTreeSet, + collections::{BTreeSet, HashMap}, fmt::{Debug, Display, Formatter}, iter, ops::Deref, @@ -37,11 +37,10 @@ use tari_ootle_common_types::{ NodeHeight, NumPreshards, ShardGroup, - ToSubstateAddress, VersionedSubstateId, VersionedSubstateIdRef, }; -use tari_state_tree::{compute_proof_for_hashes, SparseMerkleProofExt, StateTreeError, TreeHash}; +use tari_state_tree::{compute_proof_for_hashes, SparseMerkleProofExt, StateTreeError, TreeHash, Version}; use tari_template_lib::{prelude::SchnorrSignatureBytes, types::crypto::RistrettoPublicKeyBytes}; use tari_transaction::TransactionId; use time::PrimitiveDateTime; @@ -55,14 +54,21 @@ use super::{ ForeignProposalRecord, MintConfidentialOutputAtom, PendingShardStateTreeDiff, - SubstateChange, SubstateDestroyedProof, SubstateRecord, + SubstateTransitionData, TransactionAtom, ValidatorStatsUpdate, }; use crate::{ - consensus_models::{block_header::BlockHeader, Command, SubstateCreatedProof, SubstateUpdate, TransactionRecord}, + consensus_models::{ + block_header::BlockHeader, + substate_update_batch::SubstateUpdateBatch, + Command, + SubstateCreatedProof, + SubstateUpdateProof, + TransactionRecord, + }, StateStoreReadTransaction, StateStoreWriteTransaction, StorageError, @@ -74,6 +80,8 @@ const LOG_TARGET: &str = "tari::ootle::storage::consensus_models::block"; pub enum BlockError { #[error("Error computing command merkle hash: {0}")] StateTreeError(#[from] StateTreeError), + #[error("Invalid shard state versions: {details}")] + InvalidShardStateVersions { details: String }, #[error("Merke proof generation command index out of bounds: {index}/{len}")] MerkleProofGenerationCommandIndexOutOfBounds { index: usize, len: usize }, } @@ -577,61 +585,107 @@ impl Block { tx.blocks_delete(block_id) } - pub fn commit_diff(&self, tx: &mut TTx, commit_qc_id: &QcId, block_diff: BlockDiff) -> Result<(), StorageError> + pub fn commit_block_without_state_changes(&self, tx: &mut TTx, commit_qc_id: &QcId) -> Result<(), StorageError> where TTx: StateStoreWriteTransaction + Deref, TTx::Target: StateStoreReadTransaction, { - if block_diff.block_id() != self.id() { - return Err(StorageError::QueryError { - reason: format!( - "[commit_diff] Block ID mismatch. Expected: {}, got: {}", - self.id(), - block_diff.block_id() - ), - }); - } + self.commit_block(tx, commit_qc_id, &HashMap::new()) + } - if self.is_dummy() && !block_diff.is_empty() { - return Err(StorageError::QueryError { - reason: format!( - "[commit_diff] Dummy block cannot have any substate changes. Block ID: {}", - self.id() - ), - }); - } + pub fn commit_block( + &self, + tx: &mut TTx, + commit_qc_id: &QcId, + version_updates: &HashMap, + ) -> Result<(), StorageError> + where + TTx: StateStoreWriteTransaction + Deref, + TTx::Target: StateStoreReadTransaction, + { + // Set the QC that caused this block to be committed, marking it as committed + tx.blocks_set_qcs(self.id(), Some(commit_qc_id), None)?; - if !self.is_dummy() { - block_diff.remove(tx)?; + if self.is_dummy() { + info!( + target: LOG_TARGET, + "🍼 COMMIT dummy block {}", self + ); + return Ok(()); } - let BlockDiff { changes, .. } = block_diff; - - let justify_qc_id = self.justify().calculate_id(); - - for change in changes { - match change { - SubstateChange::Up { id, shard, substate } => { - let version = substate.version(); - SubstateRecord::new( - id, - version, - substate.into_substate_value(), - shard, - self.epoch(), - *self.id(), - justify_qc_id, - ) - .create(tx)?; - }, - SubstateChange::Down { id, shard } => { - SubstateRecord::destroy(tx, id, shard, self.epoch(), self.height(), &justify_qc_id)?; - }, - } - } + let Some(block_diff) = self.get_diff(&**tx).optional()? else { + info!( + target: LOG_TARGET, + "🌳 COMMIT block {} with no substate change(s)", self + ); + + // No diff to commit + return Ok(()); + }; + + // Consume the block diff + block_diff.remove(tx)?; + + info!( + target: LOG_TARGET, + "🌳 COMMIT block {} with {} substate change(s)", self, block_diff.len() + ); + + let changes = block_diff.into_changes(); + + let updates = changes.into_iter() + .filter(|change| { + if self.shard_group().contains_or_global(&change.shard()) { + true + } else { + // This should have been filtered out already, but just in case + warn!( + target: LOG_TARGET, + "❓️ Skipping substate change {} for shard {} in block {} because it is not in the shard group {}", + change.as_change_string(), + change.shard(), + self.id(), + self.shard_group() + ); + false + } + }) + // Group by shard + .try_fold(IndexMap::new(), |mut acc, change| { + let Some(state_version) = + version_updates + .get(&change.shard()) + .copied() else { + // A panic may be more appropriate here, this should never happen + return Err(StorageError::DataInconsistency { + details: format!( + "NEVER HAPPEN: Shard state version for shard {} not found in block {}", + change.shard(), + self.id() + ), + }); + }; + + let data_mut = acc.entry(change.shard()) + .or_insert_with(|| { + SubstateTransitionData { + state_version, + transitions: vec![], + } + }); + data_mut.transitions.push(change.into_transition()); + + Ok(acc) + })?; + + let batch = SubstateUpdateBatch { + epoch: self.epoch(), + updates, + }; + + SubstateRecord::commit_batch(tx, batch)?; - // Set the QC that caused this block to be committed, marking it as committed - tx.blocks_set_qcs(self.id(), Some(commit_qc_id), None)?; Ok(()) } @@ -732,7 +786,7 @@ impl Block { &self, tx: &TTx, num_preshards: NumPreshards, - ) -> Result, StorageError> { + ) -> Result, StorageError> { let committed = self .commands() .iter() @@ -752,10 +806,7 @@ impl Block { let outputs = outputs .iter() .map(|lock| lock.versioned_substate_id().as_ref()) - .filter(|id| { - self.shard_group() - .contains_or_global(&id.to_substate_address().to_shard(num_preshards)) - }); + .filter(|id| self.shard_group().contains_or_global(&id.to_shard(num_preshards))); let substates = SubstateRecord::get_all(tx, outputs)?; for substate in substates { @@ -773,13 +824,13 @@ impl Block { // substate: substate.try_into()?, // })); // } else { - updates.push(SubstateUpdate::Destroy(SubstateDestroyedProof { + updates.push(SubstateUpdateProof::Destroy(SubstateDestroyedProof { substate_id: substate.substate_id.clone(), version: substate.version, // justify: ProposalCertificate::get(tx, &destroyed.justify)?, })); } else { - updates.push(SubstateUpdate::Create(SubstateCreatedProof { + updates.push(SubstateUpdateProof::Create(SubstateCreatedProof { // created_qc: substate.get_created_quorum_certificate(tx)?, substate: substate.into(), })); diff --git a/crates/storage/src/consensus_models/block_diff.rs b/crates/storage/src/consensus_models/block_diff.rs index 12c4075e05..01f49adf18 100644 --- a/crates/storage/src/consensus_models/block_diff.rs +++ b/crates/storage/src/consensus_models/block_diff.rs @@ -5,7 +5,7 @@ use std::fmt::Debug; use tari_consensus_types::BlockId; use tari_engine_types::substate::SubstateId; -use tari_ootle_common_types::{committee::CommitteeInfo, VersionedSubstateIdRef}; +use tari_ootle_common_types::{ShardGroup, VersionedSubstateIdRef}; use crate::{ consensus_models::substate_change::SubstateChange, @@ -16,7 +16,7 @@ use crate::{ #[derive(Debug, Clone)] pub struct BlockDiff { - pub block_id: BlockId, + block_id: BlockId, pub changes: Vec, } @@ -37,13 +37,13 @@ impl BlockDiff { self.changes.is_empty() } - pub fn into_filtered(self, committee: &CommitteeInfo) -> Self { + pub fn into_filtered(self, shard_group: ShardGroup) -> Self { Self { block_id: self.block_id, changes: self .changes .into_iter() - .filter(|change| committee.shard_group().contains_or_global(&change.shard())) + .filter(|change| shard_group.contains_or_global(&change.shard())) .collect(), } } diff --git a/crates/storage/src/consensus_models/block_header.rs b/crates/storage/src/consensus_models/block_header.rs index 88a55b1227..1a77313c2a 100644 --- a/crates/storage/src/consensus_models/block_header.rs +++ b/crates/storage/src/consensus_models/block_header.rs @@ -37,7 +37,7 @@ use crate::consensus_models::Command; ts(export, export_to = "../../bindings/src/types/") )] pub struct BlockHeader { - /// "Cached" block ID/hash. This can be computed from the contents of the block header, + /// "Cached" block ID/hash. This is computed from the contents of the block header. #[cfg_attr(feature = "ts", ts(type = "string"))] id: BlockId, /// Network this block belongs to. @@ -56,7 +56,6 @@ pub struct BlockHeader { /// Shard group that created this block. shard_group: ShardGroup, /// The public key of the proposer. - #[cfg_attr(feature = "ts", ts(type = "string"))] proposed_by: RistrettoPublicKeyBytes, /// The total leader fee for this block. This should match the sum of the leader fees in the block's body. #[cfg_attr(feature = "ts", ts(type = "number"))] @@ -191,45 +190,10 @@ impl BlockHeader { .expect("Infallible with empty commands") } - #[allow(clippy::too_many_arguments)] - pub fn load( - id: BlockId, - network: Network, - parent: BlockId, - justify_id: QcId, - height: NodeHeight, - epoch: Epoch, - shard_group: ShardGroup, - proposed_by: RistrettoPublicKeyBytes, - state_merkle_root: FixedHash, - total_leader_fee: u64, - signature: Option, - timestamp: u64, - epoch_hash: FixedHash, - extra_data: ExtraData, - command_merkle_root: FixedHash, - ) -> Self { - Self { - id, - network, - parent, - justify_id, - height, - epoch, - shard_group, - proposed_by, - state_merkle_root, - command_merkle_root, - total_leader_fee, - signature, - timestamp, - epoch_hash, - extra_data, - } - } - /// This is the parent block for all genesis blocks. Its block ID is always zero. + // TODO: do we need a zero block anymore? pub fn zero_block(network: Network, num_preshards: NumPreshards) -> Self { + let shard_group = ShardGroup::all_shards(num_preshards); Self { network, id: BlockId::zero(), @@ -238,7 +202,7 @@ impl BlockHeader { .calculate_id(), height: NodeHeight::zero(), epoch: Epoch::zero(), - shard_group: ShardGroup::all_shards(num_preshards), + shard_group, proposed_by: RistrettoPublicKeyBytes::default(), state_merkle_root: FixedHash::zero(), command_merkle_root: FixedHash::zero(), diff --git a/crates/storage/src/consensus_models/epoch_checkpoint.rs b/crates/storage/src/consensus_models/epoch_checkpoint.rs index 982687023e..b9b69c9fb8 100644 --- a/crates/storage/src/consensus_models/epoch_checkpoint.rs +++ b/crates/storage/src/consensus_models/epoch_checkpoint.rs @@ -11,7 +11,13 @@ use tari_common_types::types::{CompressedPublicKey, FixedHash}; use tari_crypto::tari_utilities::ByteArray; use tari_ootle_common_types::{shard::Shard, Epoch, ShardGroup, VotePower}; use tari_sidechain::{CommandCommitProof, SidechainBlockHeader, SidechainProofValidationError, ToCommand}; -use tari_state_tree::{compute_merkle_root_for_hashes, StateTreeError, TreeHash, SPARSE_MERKLE_PLACEHOLDER_HASH}; +use tari_state_tree::{ + compute_merkle_root_for_hashes, + StateTreeError, + TreeHash, + Version, + SPARSE_MERKLE_PLACEHOLDER_HASH, +}; use tari_template_lib::prelude::RistrettoPublicKeyBytes; use crate::{StateStoreReadTransaction, StateStoreWriteTransaction, StorageError}; @@ -19,12 +25,24 @@ use crate::{StateStoreReadTransaction, StateStoreWriteTransaction, StorageError} #[derive(Debug, Clone, Serialize, Deserialize)] pub struct EpochCheckpoint { proof: CommandCommitProof, - shard_roots: IndexMap, + shard_tree_summary: IndexMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TreeRootSummary { + pub root_hash: TreeHash, + pub state_version: Version, } impl EpochCheckpoint { - pub fn new(proof: CommandCommitProof, shard_roots: IndexMap) -> Self { - Self { proof, shard_roots } + pub fn new( + proof: CommandCommitProof, + shard_tree_summary: IndexMap, + ) -> Self { + Self { + proof, + shard_tree_summary, + } } pub fn proof(&self) -> &CommandCommitProof { @@ -43,17 +61,24 @@ impl EpochCheckpoint { Epoch(self.proof.header().epoch) } - pub fn shard_roots(&self) -> &IndexMap { - &self.shard_roots + pub fn shard_tree_summary(&self) -> &IndexMap { + &self.shard_tree_summary } pub fn get_shard_root(&self, shard: Shard) -> TreeHash { - self.shard_roots + self.shard_tree_summary .get(&shard) - .copied() + .map(|summary| summary.root_hash) .unwrap_or(SPARSE_MERKLE_PLACEHOLDER_HASH) } + pub fn get_shard_state_version(&self, shard: Shard) -> Version { + self.shard_tree_summary + .get(&shard) + .map(|summary| summary.state_version) + .unwrap_or_default() + } + pub fn compute_state_merkle_root(&self) -> Result { let shard_group = self.checked_shard_group()?; let hashes = iter::once(Shard::global()) @@ -114,10 +139,10 @@ impl EpochCheckpoint { } // 1 + for global shard - if self.shard_roots().len() > num_shards + 1 { + if self.shard_tree_summary.len() > num_shards + 1 { return Err( EpochCheckpointValidationError::NumberOfShardStateRootsExceedsNumberOfShards { - num_shard_state_roots: self.shard_roots().len(), + num_shard_state_roots: self.shard_tree_summary.len(), num_shards, }, ); @@ -154,7 +179,7 @@ impl Display for EpochCheckpoint { "EpochCheckpoint: block_id={}, epoch={}, count(shard_roots)={}", self.proof.header().calculate_block_id(), self.proof.header().epoch, - self.shard_roots.len() + self.shard_tree_summary.len() ) } } diff --git a/crates/storage/src/consensus_models/epoch_state_root.rs b/crates/storage/src/consensus_models/epoch_state_root.rs deleted file mode 100644 index 4873ea2e16..0000000000 --- a/crates/storage/src/consensus_models/epoch_state_root.rs +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2025 The Tari Project -// SPDX-License-Identifier: BSD-3-Clause - -use log::*; -use serde::{Deserialize, Serialize}; -use tari_ootle_common_types::{Epoch, ShardGroup}; -use tari_state_tree::TreeHash; - -use crate::{StateStoreReadTransaction, StateStoreWriteTransaction, StorageError}; - -const LOG_TARGET: &str = "tari::ootle::consensus::epoch_state_root"; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct EpochStateRoot { - /// The epoch that applies to this state root - pub epoch: Epoch, - /// The shard group that applies to this state root - pub shard_group: ShardGroup, - /// The state root of the epoch - pub state_root: TreeHash, -} - -impl EpochStateRoot { - pub fn new(epoch: Epoch, shard_group: ShardGroup, state_root: TreeHash) -> Self { - Self { - epoch, - shard_group, - state_root, - } - } - - pub fn epoch(&self) -> Epoch { - self.epoch - } - - pub fn shard_group(&self) -> &ShardGroup { - &self.shard_group - } - - pub fn state_root(&self) -> TreeHash { - self.state_root - } -} - -impl EpochStateRoot { - pub fn set(&self, tx: &mut TTx) -> Result<(), StorageError> { - info!( - target: LOG_TARGET, - "Setting epoch state root for epoch {} and shard group {} to {}", - self.epoch, - self.shard_group, - self.state_root, - ); - tx.previous_epoch_state_root_set(self) - } - - pub fn get(tx: &TTx) -> Result { - tx.previous_epoch_state_root_get() - } -} diff --git a/crates/storage/src/consensus_models/mod.rs b/crates/storage/src/consensus_models/mod.rs index f876656bd1..3c308a1965 100644 --- a/crates/storage/src/consensus_models/mod.rs +++ b/crates/storage/src/consensus_models/mod.rs @@ -10,7 +10,6 @@ mod burnt_utxo; mod command; mod commands_commit_proof; mod epoch_checkpoint; -mod epoch_state_root; mod evidence; mod foreign_parked_proposal; mod foreign_proposal; @@ -23,6 +22,7 @@ mod state_tree_diff; mod substate; mod substate_change; mod substate_lock; +mod substate_update_batch; mod transaction; mod transaction_execution; mod transaction_pool; @@ -39,7 +39,6 @@ pub use burnt_utxo::*; pub use command::*; pub use commands_commit_proof::*; pub use epoch_checkpoint::*; -pub use epoch_state_root::*; pub use evidence::*; pub use foreign_parked_proposal::*; pub use foreign_proposal::*; @@ -52,6 +51,7 @@ pub use state_tree_diff::*; pub use substate::*; pub use substate_change::*; pub use substate_lock::*; +pub use substate_update_batch::*; pub use transaction::*; pub use transaction_execution::*; pub use transaction_pool::*; diff --git a/crates/storage/src/consensus_models/state_transition.rs b/crates/storage/src/consensus_models/state_transition.rs index 77c4addb15..3f913dd30d 100644 --- a/crates/storage/src/consensus_models/state_transition.rs +++ b/crates/storage/src/consensus_models/state_transition.rs @@ -1,159 +1,57 @@ // Copyright 2024 The Tari Project // SPDX-License-Identifier: BSD-3-Clause -use std::{ - fmt::{Display, Formatter}, - io::{Read, Write}, - mem, -}; - use serde::{Deserialize, Serialize}; use tari_ootle_common_types::{shard::Shard, Epoch}; -use tari_state_tree::{SubstateTreeChange, Version}; +use tari_state_tree::Version; -use crate::{consensus_models::SubstateUpdate, StateStoreReadTransaction, StorageError}; +use crate::{consensus_models::SubstateUpdateProof, StateStoreReadTransaction, StorageError}; #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StateTransition { - pub id: StateTransitionId, +pub struct StateVersionTransitions { + pub epoch: Epoch, + pub shard: Shard, pub state_version: Version, - pub update: SubstateUpdate, -} - -impl StateTransition { - pub fn to_tree_change(&self) -> SubstateTreeChange { - match &self.update { - SubstateUpdate::Create(create) => { - let id = create.substate.as_versioned_substate_id_ref(); - SubstateTreeChange::Up { - id: id.to_owned(), - value_hash: create.substate.to_value_hash(), - } - }, - SubstateUpdate::Destroy(destroy) => SubstateTreeChange::Down { - id: destroy.to_versioned_substate_id(), - }, + pub updates: Vec, +} + +impl StateVersionTransitions { + pub fn into_chunks(mut self, size: usize) -> Vec { + let num_chunks = self.updates.len().div_ceil(size); + let mut chunks = Vec::with_capacity(num_chunks); + loop { + if self.updates.len() < size { + chunks.push(Self { + epoch: self.epoch, + shard: self.shard, + state_version: self.state_version, + updates: self.updates, + }); + break; + } + + let chunk = self.updates.split_off(size); + + chunks.push(Self { + epoch: self.epoch, + shard: self.shard, + state_version: self.state_version, + updates: chunk, + }); } + chunks } } -impl StateTransition { - pub fn get_n_after( - tx: &TTx, - n: usize, - after_id: StateTransitionId, - end_epoch: Epoch, - ) -> Result, StorageError> { - tx.state_transitions_get_n_after(n, after_id, end_epoch) - } +pub struct StateTransition; - pub fn get_last_id( +impl StateTransition { + pub fn get_for_shard( tx: &TTx, shard: Shard, - ) -> Result { - tx.state_transitions_get_last_id(shard) - } -} - -impl Display for StateTransition { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}: {}", self.id, self.update) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub struct StateTransitionId { - epoch: Epoch, - shard: Shard, - seq: u64, -} -impl StateTransitionId { - const BYTE_SIZE: usize = mem::size_of::(); - - pub fn new(epoch: Epoch, shard: Shard, seq: u64) -> Self { - Self { epoch, shard, seq } - } - - pub fn initial(shard: Shard) -> Self { - Self::new(Epoch(1), shard, 0) - } - - pub fn from_bytes(mut bytes: &[u8]) -> Option { - if bytes.len() < Self::BYTE_SIZE { - return None; - } - let bytes_mut = &mut bytes; - let epoch = Epoch(u64::from_be_bytes(copy_fixed(bytes_mut))); - let shard = Shard::from(u32::from_be_bytes(copy_fixed(bytes_mut))); - let seq = u64::from_be_bytes(copy_fixed(bytes_mut)); - Some(Self::new(epoch, shard, seq)) - } - - pub fn as_bytes(&self) -> [u8; Self::BYTE_SIZE] { - let mut buf = [0u8; Self::BYTE_SIZE]; - let buf_mut = &mut buf.as_mut_slice(); - write_fixed(self.epoch.to_be_bytes(), buf_mut); - write_fixed(self.shard.as_u32().to_be_bytes(), buf_mut); - write_fixed(self.seq.to_be_bytes(), buf_mut); - buf - } - - pub fn epoch(&self) -> Epoch { - self.epoch - } - - pub fn shard(&self) -> Shard { - self.shard - } - - pub fn seq(self) -> u64 { - self.seq - } -} - -impl Display for StateTransitionId { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "state transition ({}, seq = {}, {})", - self.shard(), - self.seq(), - self.epoch(), - ) - } -} - -/// Copies bytes into a fixed byte array. -/// -/// ## Panics -/// Caller must ensure that sufficient bytes remain on the mut ref to the input slice. -fn copy_fixed(bytes: &mut &[u8]) -> [u8; SZ] { - let mut buf = [0u8; SZ]; - bytes - .read_exact(&mut buf) - .expect("copy_fixed: Expected enough bytes to read"); - buf -} - -/// Writes fixed bytes into a buffer. -/// ## Panics -/// Caller must ensure that the buffer has sufficient space for the fixed bytes. -fn write_fixed(buf: [u8; SZ], out: &mut &mut [u8]) { - out.write_all(&buf) - .expect("write_fixed: Expected buffer to have sufficient space for fixed bytes"); -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn to_and_from_bytes() { - let id = StateTransitionId::new(Epoch(1), Shard::from(2), 3); - let bytes = id.as_bytes(); - let id2 = StateTransitionId::from_bytes(&bytes).unwrap(); - assert_eq!(id, id2); - - assert_eq!(StateTransitionId::from_bytes(&[1, 2, 3]), None); + state_version: Version, + include_values: bool, + ) -> Result { + tx.state_transitions_get_after(shard, state_version, include_values) } } diff --git a/crates/storage/src/consensus_models/state_tree_diff.rs b/crates/storage/src/consensus_models/state_tree_diff.rs index 51f6a2b17f..f08407e150 100644 --- a/crates/storage/src/consensus_models/state_tree_diff.rs +++ b/crates/storage/src/consensus_models/state_tree_diff.rs @@ -10,22 +10,22 @@ use indexmap::IndexMap; use serde::{Deserialize, Serialize}; use tari_consensus_types::BlockId; use tari_ootle_common_types::shard::Shard; -use tari_state_tree::{StateHashTreeDiff, Version}; +use tari_state_tree::{StateHashTreeDiff, StateTreePayload, Version}; use crate::{StateStoreReadTransaction, StateStoreWriteTransaction, StorageError}; -#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct PendingShardStateTreeDiff { pub version: Version, - pub diff: StateHashTreeDiff, + pub diff: StateHashTreeDiff, } impl PendingShardStateTreeDiff { - pub fn new(version: Version, diff: StateHashTreeDiff) -> Self { + pub fn new(version: Version, diff: StateHashTreeDiff) -> Self { Self { version, diff } } - pub fn load(version: Version, diff: StateHashTreeDiff) -> Self { + pub fn load(version: Version, diff: StateHashTreeDiff) -> Self { Self { version, diff } } } diff --git a/crates/storage/src/consensus_models/substate.rs b/crates/storage/src/consensus_models/substate.rs index f7af8d775a..458368e86b 100644 --- a/crates/storage/src/consensus_models/substate.rs +++ b/crates/storage/src/consensus_models/substate.rs @@ -5,7 +5,7 @@ use std::{collections::HashSet, fmt, fmt::Display}; use serde::{Deserialize, Serialize}; use tari_common_types::types::FixedHash; -use tari_consensus_types::{BlockId, LeafBlock, ProposalCertificate, QcId}; +use tari_consensus_types::LeafBlock; use tari_engine_types::{ serde_with, substate::{hash_substate, Substate, SubstateId, SubstateValue}, @@ -14,15 +14,20 @@ use tari_ootle_common_types::{ displayable::Displayable, shard::Shard, Epoch, - NodeHeight, SubstateAddress, SubstateRequirement, VersionedSubstateId, VersionedSubstateIdRef, }; +use tari_state_tree::{SubstateTreeChange, Version}; use tari_transaction::TransactionId; -use crate::{consensus_models::SubstateLock, StateStoreReadTransaction, StateStoreWriteTransaction, StorageError}; +use crate::{ + consensus_models::{substate_update_batch::SubstateUpdateBatch, SubstateLock, SubstateTransition}, + StateStoreReadTransaction, + StateStoreWriteTransaction, + StorageError, +}; #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr( @@ -37,39 +42,16 @@ pub struct SubstateRecord { #[cfg_attr(feature = "ts", ts(type = "string"))] #[serde(with = "serde_with::hex")] pub state_hash: FixedHash, - #[cfg_attr(feature = "ts", ts(type = "string"))] - pub created_justify: QcId, - #[cfg_attr(feature = "ts", ts(type = "string"))] - pub created_block: BlockId, - pub created_by_shard: Shard, - pub created_at_epoch: Epoch, + pub created: SubstateCreated, pub destroyed: Option, } -#[derive(Debug, Clone, Serialize, Deserialize)] -#[cfg_attr( - feature = "ts", - derive(ts_rs::TS), - ts(export, export_to = "../../bindings/src/types/") -)] -pub struct SubstateDestroyed { - #[cfg_attr(feature = "ts", ts(type = "string"))] - pub justify: QcId, - #[cfg_attr(feature = "ts", ts(type = "string"))] - pub by_block: NodeHeight, - pub at_epoch: Epoch, - pub by_shard: Shard, -} - impl SubstateRecord { pub fn new>( substate_id: SubstateId, version: u32, value: V, - created_by_shard: Shard, - created_at_epoch: Epoch, - created_block: BlockId, - created_justify: QcId, + created: SubstateCreated, ) -> Self { let value = value.into(); Self { @@ -77,10 +59,7 @@ impl SubstateRecord { version, state_hash: value.to_value_hash(version), substate_value: value.into_value(), - created_justify, - created_by_shard, - created_at_epoch, - created_block, + created, destroyed: None, } } @@ -117,16 +96,25 @@ impl SubstateRecord { Some(Substate::new(self.version, self.substate_value?)) } + pub fn into_substate_value_or_hash(self) -> SubstateValueOrHash { + self.substate_value + .map(Into::into) + .unwrap_or_else(|| self.state_hash.into()) + } + pub fn version(&self) -> u32 { self.version } - pub fn created_block(&self) -> BlockId { - self.created_block + /// Returns the shard this substate is in. + /// WARN: you cant trust this if this is deserialized from an untrusted source, this should be validated by locally + /// calculating which shard this substate falls. + pub fn shard(&self) -> Shard { + self.created.in_shard } - pub fn created_justify(&self) -> &QcId { - &self.created_justify + pub fn created(&self) -> &SubstateCreated { + &self.created } pub fn destroyed(&self) -> Option<&SubstateDestroyed> { @@ -144,6 +132,23 @@ impl SubstateRecord { pub fn state_hash(&self) -> &FixedHash { &self.state_hash } + + pub fn into_transition(self) -> SubstateTransition { + if self.is_up() { + SubstateTransition::Up { + id: self.substate_id, + version: self.version, + substate_or_hash: self + .substate_value + .map(Into::into) + .unwrap_or_else(|| self.state_hash.into()), + } + } else { + SubstateTransition::Down { + id: VersionedSubstateId::new(self.substate_id, self.version), + } + } + } } impl SubstateRecord { @@ -166,8 +171,11 @@ impl SubstateRecord { tx.substate_locks_remove_many_for_transactions(transaction_ids) } - pub fn create(&self, tx: &mut TTx) -> Result<(), StorageError> { - tx.substates_create(self)?; + pub fn commit_batch( + tx: &mut TTx, + updates: SubstateUpdateBatch, + ) -> Result<(), StorageError> { + tx.substates_commit_batch(updates)?; Ok(()) } @@ -258,39 +266,6 @@ impl SubstateRecord { Ok(rec) } - pub fn get_created_proposal_certificate( - &self, - tx: &TTx, - ) -> Result { - tx.proposal_certificates_get(self.created_at_epoch, self.created_justify()) - } - - pub fn get_destroyed_proposal_certificate( - &self, - tx: &TTx, - ) -> Result, StorageError> { - self.destroyed() - .map(|destroyed| tx.proposal_certificates_get(destroyed.at_epoch, &destroyed.justify)) - .transpose() - } - - pub fn destroy( - tx: &mut TTx, - versioned_substate_id: VersionedSubstateId, - shard: Shard, - epoch: Epoch, - destroyed_by_block: NodeHeight, - destroyed_justify: &QcId, - ) -> Result<(), StorageError> { - tx.substates_down( - versioned_substate_id, - shard, - epoch, - destroyed_by_block, - destroyed_justify, - ) - } - pub fn prune_downed_values( tx: &mut TTx, epoch: Epoch, @@ -318,6 +293,32 @@ impl SubstateDestroyedProof { } } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr( + feature = "ts", + derive(ts_rs::TS), + ts(export, export_to = "../../bindings/src/types/") +)] +pub struct SubstateCreated { + // TODO: consider removing this field, it's not used + pub at_epoch: Epoch, + // Note: This field not strictly necessary, since the shard can be derived from (SubstateId, Version) and + // NumPreshards. But the cost is negligible, and it makes the metadata more self-contained. + pub in_shard: Shard, + pub at_state_version: Version, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr( + feature = "ts", + derive(ts_rs::TS), + ts(export, export_to = "../../bindings/src/types/") +)] +pub struct SubstateDestroyed { + pub at_epoch: Epoch, + pub at_state_version: Version, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub enum SubstateValueOrHash { Value(Box), @@ -405,12 +406,12 @@ impl From for SubstateData { } #[derive(Debug, Clone, Serialize, Deserialize)] -pub enum SubstateUpdate { +pub enum SubstateUpdateProof { Create(SubstateCreatedProof), Destroy(SubstateDestroyedProof), } -impl SubstateUpdate { +impl SubstateUpdateProof { pub fn is_create(&self) -> bool { matches!(self, Self::Create(_)) } @@ -443,15 +444,30 @@ impl SubstateUpdate { _ => None, } } + + pub fn to_tree_change(&self) -> SubstateTreeChange { + match self { + Self::Create(create) => { + let id = create.substate.as_versioned_substate_id_ref(); + SubstateTreeChange::Up { + id: id.to_owned(), + value_hash: create.substate.to_value_hash(), + } + }, + Self::Destroy(destroy) => SubstateTreeChange::Down { + id: destroy.to_versioned_substate_id(), + }, + } + } } -impl From for SubstateUpdate { +impl From for SubstateUpdateProof { fn from(value: SubstateCreatedProof) -> Self { Self::Create(value) } } -impl Display for SubstateUpdate { +impl Display for SubstateUpdateProof { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Create(proof) => write!(f, "Create: {}(v{})", proof.substate.substate_id, proof.substate.version), diff --git a/crates/storage/src/consensus_models/substate_change.rs b/crates/storage/src/consensus_models/substate_change.rs index 53d743a012..3e6e09a774 100644 --- a/crates/storage/src/consensus_models/substate_change.rs +++ b/crates/storage/src/consensus_models/substate_change.rs @@ -14,6 +14,8 @@ use tari_ootle_common_types::{ }; use tari_state_tree::SubstateTreeChange; +use crate::consensus_models::SubstateTransition; + #[derive(Debug, Clone, Serialize, Deserialize)] pub enum SubstateChange { Up { @@ -60,6 +62,7 @@ impl SubstateChange { } } + /// A cached shard value. This can be calculated from the substate address, but is cached here for performance. pub fn shard(&self) -> Shard { match self { SubstateChange::Up { shard, .. } => *shard, @@ -95,6 +98,17 @@ impl SubstateChange { SubstateChange::Down { .. } => "Down", } } + + pub fn into_transition(self) -> SubstateTransition { + match self { + SubstateChange::Up { id, substate, .. } => SubstateTransition::Up { + id, + version: substate.version(), + substate_or_hash: substate.into_substate_value().into(), + }, + SubstateChange::Down { id, .. } => SubstateTransition::Down { id }, + } + } } impl Display for SubstateChange { diff --git a/crates/storage/src/consensus_models/substate_update_batch.rs b/crates/storage/src/consensus_models/substate_update_batch.rs new file mode 100644 index 0000000000..f2a6bc7f22 --- /dev/null +++ b/crates/storage/src/consensus_models/substate_update_batch.rs @@ -0,0 +1,55 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use indexmap::IndexMap; +use tari_engine_types::substate::SubstateId; +use tari_ootle_common_types::{shard::Shard, Epoch, VersionedSubstateId}; +use tari_state_tree::Version; + +use crate::consensus_models::SubstateValueOrHash; + +pub struct SubstateUpdateBatch { + pub epoch: Epoch, + pub updates: IndexMap, +} + +impl SubstateUpdateBatch { + pub fn new(epoch: Epoch) -> Self { + Self { + epoch, + updates: IndexMap::new(), + } + } + + pub fn add_transition( + &mut self, + shard: Shard, + state_version: Version, + transition: SubstateTransition, + ) -> &mut Self { + self.updates + .entry(shard) + .or_insert_with(|| SubstateTransitionData { + state_version, + transitions: Vec::new(), + }) + .transitions + .push(transition); + self + } +} + +pub struct SubstateTransitionData { + pub state_version: Version, + pub transitions: Vec, +} +pub enum SubstateTransition { + Up { + id: SubstateId, + version: u32, + substate_or_hash: SubstateValueOrHash, + }, + Down { + id: VersionedSubstateId, + }, +} diff --git a/crates/storage/src/state_store/mod.rs b/crates/storage/src/state_store/mod.rs index b77a912c23..311579af23 100644 --- a/crates/storage/src/state_store/mod.rs +++ b/crates/storage/src/state_store/mod.rs @@ -31,12 +31,12 @@ use tari_ootle_common_types::{ NodeHeight, NumPreshards, ShardGroup, + ShardStateVersions, SubstateAddress, ToSubstateAddress, - VersionedSubstateId, VersionedSubstateIdRef, }; -use tari_state_tree::{Node, NodeKey, StaleTreeNode, Version}; +use tari_state_tree::{Node, NodeKey, StaleTreeNode, StateTreePayload, Version}; use tari_template_lib::{models::UnclaimedConfidentialOutputAddress, types::crypto::RistrettoPublicKeyBytes}; use tari_transaction::TransactionId; use time::PrimitiveDateTime; @@ -48,7 +48,6 @@ use crate::{ BlockTransactionExecution, BurntUtxo, EpochCheckpoint, - EpochStateRoot, Evidence, ForeignParkedProposal, ForeignProposal, @@ -58,12 +57,12 @@ use crate::{ LockedSubstateValue, NoVoteReason, PendingShardStateTreeDiff, - StateTransition, - StateTransitionId, + StateVersionTransitions, SubstateChange, SubstateLock, SubstatePledges, SubstateRecord, + SubstateUpdateBatch, TransactionExecution, TransactionPoolRecord, TransactionPoolStage, @@ -190,6 +189,7 @@ pub trait StateStoreReadTransaction: Sized { ) -> Result; fn block_diffs_get(&self, block_id: &BlockId) -> Result; + fn block_diffs_get_last_change_for_substate( &self, block_id: &BlockId, @@ -275,23 +275,31 @@ pub trait StateStoreReadTransaction: Sized { block_id: &BlockId, ) -> Result>, StorageError>; - fn state_transitions_get_n_after( + // -------------------------------- State transitions -------------------------------- // + fn state_transitions_get_after( &self, - n: usize, - id: StateTransitionId, - end_epoch: Epoch, - ) -> Result, StorageError>; + shard: Shard, + state_version: Version, + include_values: bool + ) -> Result; - fn state_transitions_get_last_id(&self, shard: Shard) -> Result; + // -------------------------------- State Tree -------------------------------- // - fn state_tree_nodes_get(&self, shard: Shard, key: &NodeKey) -> Result, StorageError>; + fn state_tree_nodes_get(&self, shard: Shard, key: &NodeKey) -> Result, StorageError>; + fn state_tree_nodes_get_all_by_state_version( + &self, + shard: Shard, + state_version: Version, + ) -> Result)>, StorageError>; fn state_tree_versions_get_latest(&self, shard: Shard) -> Result, StorageError>; + fn state_tree_versions_get_latest_for_shard_group( + &self, + shard_group: ShardGroup, + ) -> Result; // -------------------------------- Epoch checkpoint -------------------------------- // fn epoch_checkpoint_get(&self, epoch: Epoch) -> Result; - fn previous_epoch_state_root_get(&self) -> Result; - // -------------------------------- Foreign Substate Pledges -------------------------------- // fn foreign_substate_pledges_exists_for_transaction_and_address( &self, @@ -480,15 +488,9 @@ pub trait StateStoreWriteTransaction { fn substate_locks_remove_any_by_block_id(&mut self, block_id: &BlockId) -> Result<(), StorageError>; - fn substates_create(&mut self, substate: &SubstateRecord) -> Result<(), StorageError>; - fn substates_down( - &mut self, - versioned_substate_id: VersionedSubstateId, - shard: Shard, - epoch: Epoch, - destroyed_block_height: NodeHeight, - destroyed_qc_id: &QcId, - ) -> Result<(), StorageError>; + // -------------------------------- Substates -------------------------------- // + + fn substates_commit_batch(&mut self, update_batch: SubstateUpdateBatch) -> Result<(), StorageError>; fn substates_prune_downed_values(&mut self, epoch: Epoch) -> Result<(), StorageError>; // -------------------------------- Foreign pledges -------------------------------- // @@ -522,7 +524,7 @@ pub trait StateStoreWriteTransaction { fn state_tree_nodes_batch_insert( &mut self, shard: Shard, - nodes: Vec<(NodeKey, Node)>, + nodes: Vec<(NodeKey, Node)>, ) -> Result<(), StorageError>; fn state_tree_nodes_record_stale_tree_nodes( @@ -538,9 +540,6 @@ pub trait StateStoreWriteTransaction { // -------------------------------- Epoch checkpoint -------------------------------- // fn epoch_checkpoint_save(&mut self, checkpoint: &EpochCheckpoint) -> Result<(), StorageError>; - // -------------------------------- Epoch state root -------------------------------- // - fn previous_epoch_state_root_set(&mut self, epoch_state_root: &EpochStateRoot) -> Result<(), StorageError>; - // -------------------------------- BurntUtxo -------------------------------- // fn burnt_utxos_insert(&mut self, burnt_utxo: &BurntUtxo) -> Result<(), StorageError>; fn burnt_utxos_set_proposed_block( diff --git a/utilities/db_inspector/src/webserver/handlers/bookkeeping.rs b/utilities/db_inspector/src/webserver/handlers/bookkeeping.rs index 3caf7a60e3..b9f65556ac 100644 --- a/utilities/db_inspector/src/webserver/handlers/bookkeeping.rs +++ b/utilities/db_inspector/src/webserver/handlers/bookkeeping.rs @@ -85,14 +85,14 @@ pub async fn list( )?; add_item( &tx, - "commit_block", - column_families::bookkeeping::CommitBlockCf, + "highest_seen_block", + column_families::bookkeeping::HighestSeenBlockCf, &mut table, )?; add_item( &tx, - "previous_epoch_root", - column_families::bookkeeping::PreviousEpochStateRootCf, + "commit_block", + column_families::bookkeeping::CommitBlockCf, &mut table, )?; add_item( diff --git a/utilities/db_inspector/src/webserver/handlers/state_transitions.rs b/utilities/db_inspector/src/webserver/handlers/state_transitions.rs index 351f044467..2eebdc5346 100644 --- a/utilities/db_inspector/src/webserver/handlers/state_transitions.rs +++ b/utilities/db_inspector/src/webserver/handlers/state_transitions.rs @@ -29,11 +29,10 @@ pub async fn list( let mut table = TableResponse::new([ Column::new("epoch", "Epoch"), Column::new("shard", "Shard"), - Column::new("seq", "Seq"), + Column::new("state_version", "State Version"), Column::new("substate_id", "Substate ID"), Column::new("version", "Version"), Column::new("transition", "Transition"), - Column::new("substate_address", "Substate Address"), ]); let tx = db.read_only_context(); @@ -56,19 +55,20 @@ pub async fn list( let page_size = req.limit.unwrap_or(1_000); let skip = req.page.unwrap_or(0) * page_size; for result in iter.skip(skip).take(page_size) { - let (id, data) = result?; - let encoded_key = cf.encode_key(&id); - let substate = substate_cf.get(&data.substate_address, OPERATION).optional()?; - table.add_row(json!({ - "id": hex::encode(encoded_key), - "epoch": id.epoch(), - "shard": id.shard(), - "seq": id.seq(), - "substate_id": substate.as_ref().map(|s| s.substate_id()), - "version": substate.as_ref().map(|s| s.version()), - "transition": data.transition, - "substate_address": data.substate_address, - })); + let ((shard, state_version), data) = result?; + let encoded_key = cf.encode_key(&(shard, state_version)); + for (i, transition) in data.transitions.into_iter().enumerate() { + let substate = substate_cf.get(&transition.substate_address, OPERATION).optional()?; + table.add_row(json!({ + "id": hex::encode(&encoded_key) + &format!("-{}", i), + "epoch": data.epoch, + "shard": shard, + "state_version": state_version, + "substate_id": substate.as_ref().map(|s| s.substate_id()), + "version": substate.as_ref().map(|s| s.version()), + "transition": transition.transition, + })); + } } let total = cf.count(OPERATION)?; table.set_total_entries(total); diff --git a/utilities/db_inspector/src/webserver/server.rs b/utilities/db_inspector/src/webserver/server.rs index dc77ab1e3a..292bf27910 100644 --- a/utilities/db_inspector/src/webserver/server.rs +++ b/utilities/db_inspector/src/webserver/server.rs @@ -98,7 +98,6 @@ pub async fn run(context: HandlerContext) -> anyhow::Result<()> { column_families::substate::HeadIndex, column_families::substate::UnprunedDownedValuesIndex, // column_families::state_transition::StateTransitionModel, - column_families::state_transition::ShardSeqIndex, // column_families::foreign_substate_pledge::ForeignSubstatePledgeModel, column_families::pending_state_tree_diff::PendingStateTreeDiffCf, column_families::state_tree::StateTreeCf, From 5b50ef7127f45a248f05fb7a1153b9f8940c4749 Mon Sep 17 00:00:00 2001 From: Stan Bondi Date: Tue, 19 Aug 2025 15:52:16 +0400 Subject: [PATCH 2/5] update bindings + review comments/bug fixes --- .../src/p2p/rpc/state_sync_task.rs | 2 +- .../src/state_bootstrap.rs | 12 +- bindings/src/index.ts | 2 + bindings/src/types/BlockHeader.ts | 5 +- bindings/src/types/ShardStateVersions.ts | 11 ++ bindings/src/types/SubstateCreated.ts | 5 + bindings/src/types/SubstateDestroyed.ts | 3 +- bindings/src/types/SubstateRecord.ts | 8 +- .../common_types/src/versioned_substate_id.rs | 5 +- crates/consensus_tests/fixtures/block.json | 4 +- .../fixtures/block_with_dummies.json | 2 +- crates/consensus_tests/src/substate_store.rs | 2 +- crates/consensus_tests/src/support/harness.rs | 11 +- crates/p2p/proto/rpc.proto | 2 +- crates/rpc_state_sync/src/state_sync.rs | 35 ++--- .../src/column_families/substate.rs | 9 +- crates/state_store_rocksdb/src/writer.rs | 137 ++++++++++-------- crates/state_store_tests/src/helpers.rs | 26 ++-- crates/state_store_tests/src/substates.rs | 10 +- crates/storage/src/consensus_models/block.rs | 22 +-- .../src/consensus_models/state_transition.rs | 25 ++-- .../storage/src/consensus_models/substate.rs | 2 +- .../consensus_models/substate_update_batch.rs | 23 +-- crates/storage/src/state_store/mod.rs | 2 +- pnpm-lock.yaml | 20 +-- .../webserver/handlers/state_transitions.rs | 42 ++++-- 26 files changed, 214 insertions(+), 213 deletions(-) create mode 100644 bindings/src/types/ShardStateVersions.ts create mode 100644 bindings/src/types/SubstateCreated.ts diff --git a/applications/tari_validator_node/src/p2p/rpc/state_sync_task.rs b/applications/tari_validator_node/src/p2p/rpc/state_sync_task.rs index b52b3c4f13..81840715da 100644 --- a/applications/tari_validator_node/src/p2p/rpc/state_sync_task.rs +++ b/applications/tari_validator_node/src/p2p/rpc/state_sync_task.rs @@ -108,7 +108,7 @@ impl StateSyncTask { } async fn send_responses(&mut self, transitions: StateVersionTransitions) -> Result<(), ()> { - let chunks = transitions.into_chunks(self.batch_size.get()); + let chunks = transitions.into_chunks(self.batch_size); let num_chunks = chunks.len(); for (i, chunk) in chunks.into_iter().enumerate() { diff --git a/applications/tari_validator_node/src/state_bootstrap.rs b/applications/tari_validator_node/src/state_bootstrap.rs index a62c2eddd5..b8febd3896 100644 --- a/applications/tari_validator_node/src/state_bootstrap.rs +++ b/applications/tari_validator_node/src/state_bootstrap.rs @@ -189,11 +189,13 @@ where let substate_id = substate_id.into(); let shard = VersionedSubstateIdRef::new(&substate_id, 0).to_shard(num_preshards); let mut batch = SubstateUpdateBatch::new(Epoch::zero()); - batch.add_transition(shard, INITIAL_STATE_VERSION, SubstateTransition::Up { - id: substate_id, - version: 0, - substate_or_hash: value.into().into(), - }); + batch + .with_transition(shard, INITIAL_STATE_VERSION) + .push(SubstateTransition::Up { + id: substate_id, + version: 0, + substate_or_hash: value.into().into(), + }); SubstateRecord::commit_batch(tx, batch)?; diff --git a/bindings/src/index.ts b/bindings/src/index.ts index ec0d040da0..04a8fb6c27 100644 --- a/bindings/src/index.ts +++ b/bindings/src/index.ts @@ -96,6 +96,7 @@ export * from "./types/Scalar32Bytes"; export * from "./types/SchnorrSignatureBytes"; export * from "./types/ShardGroupEvidence"; export * from "./types/ShardGroup"; +export * from "./types/ShardStateVersions"; export * from "./types/Shard"; export * from "./types/StealthInputsStatement"; export * from "./types/StealthInput"; @@ -103,6 +104,7 @@ export * from "./types/StealthOutputsStatement"; export * from "./types/StealthTransferStatement"; export * from "./types/StealthUnspentOutput"; export * from "./types/SubstateAddress"; +export * from "./types/SubstateCreated"; export * from "./types/SubstateDestroyed"; export * from "./types/SubstateDiff"; export * from "./types/SubstateId"; diff --git a/bindings/src/types/BlockHeader.ts b/bindings/src/types/BlockHeader.ts index f37adb6090..63f3bc2903 100644 --- a/bindings/src/types/BlockHeader.ts +++ b/bindings/src/types/BlockHeader.ts @@ -2,12 +2,13 @@ import type { Epoch } from "./Epoch"; import type { ExtraData } from "./ExtraData"; import type { NodeHeight } from "./NodeHeight"; +import type { RistrettoPublicKeyBytes } from "./RistrettoPublicKeyBytes"; import type { SchnorrSignatureBytes } from "./SchnorrSignatureBytes"; import type { ShardGroup } from "./ShardGroup"; export type BlockHeader = { /** - * "Cached" block ID/hash. This can be computed from the contents of the block header, + * "Cached" block ID/hash. This is computed from the contents of the block header. */ id: string; /** @@ -37,7 +38,7 @@ export type BlockHeader = { /** * The public key of the proposer. */ - proposed_by: string; + proposed_by: RistrettoPublicKeyBytes; /** * The total leader fee for this block. This should match the sum of the leader fees in the block's body. */ diff --git a/bindings/src/types/ShardStateVersions.ts b/bindings/src/types/ShardStateVersions.ts new file mode 100644 index 0000000000..ac6b28ef54 --- /dev/null +++ b/bindings/src/types/ShardStateVersions.ts @@ -0,0 +1,11 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * The state versions for each shard that maps each shard managed by the ShardGroup (including the + * global shard) to a state version. + * + * For example, if the ShardGroup is [1, 3], the state versions will contain 4 + * elements. The first element is always the global shard (shard 0) version. The second element is the state + * version for shard 1, third is shard 2, and forth is shard 3. + */ +export type ShardStateVersions = { inner: number[] }; diff --git a/bindings/src/types/SubstateCreated.ts b/bindings/src/types/SubstateCreated.ts new file mode 100644 index 0000000000..b7a1bbb11b --- /dev/null +++ b/bindings/src/types/SubstateCreated.ts @@ -0,0 +1,5 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { Epoch } from "./Epoch"; +import type { Shard } from "./Shard"; + +export type SubstateCreated = { at_epoch: Epoch; in_shard: Shard; at_state_version: bigint }; diff --git a/bindings/src/types/SubstateDestroyed.ts b/bindings/src/types/SubstateDestroyed.ts index 6355577ffb..2108419860 100644 --- a/bindings/src/types/SubstateDestroyed.ts +++ b/bindings/src/types/SubstateDestroyed.ts @@ -1,5 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { Epoch } from "./Epoch"; -import type { Shard } from "./Shard"; -export type SubstateDestroyed = { justify: string; by_block: string; at_epoch: Epoch; by_shard: Shard }; +export type SubstateDestroyed = { at_epoch: Epoch; at_state_version: bigint }; diff --git a/bindings/src/types/SubstateRecord.ts b/bindings/src/types/SubstateRecord.ts index 1eb0431a49..02548d06cd 100644 --- a/bindings/src/types/SubstateRecord.ts +++ b/bindings/src/types/SubstateRecord.ts @@ -1,6 +1,5 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { Epoch } from "./Epoch"; -import type { Shard } from "./Shard"; +import type { SubstateCreated } from "./SubstateCreated"; import type { SubstateDestroyed } from "./SubstateDestroyed"; import type { SubstateId } from "./SubstateId"; import type { SubstateValue } from "./SubstateValue"; @@ -10,9 +9,6 @@ export type SubstateRecord = { version: number; substate_value: SubstateValue | null; state_hash: string; - created_justify: string; - created_block: string; - created_by_shard: Shard; - created_at_epoch: Epoch; + created: SubstateCreated; destroyed: SubstateDestroyed | null; }; diff --git a/crates/common_types/src/versioned_substate_id.rs b/crates/common_types/src/versioned_substate_id.rs index 9f81983d27..49bbf2d317 100644 --- a/crates/common_types/src/versioned_substate_id.rs +++ b/crates/common_types/src/versioned_substate_id.rs @@ -72,9 +72,10 @@ impl SubstateRequirement { SubstateAddress::from_substate_id(self.substate_id(), 0) } - /// Calculates and returns the shard number that this SubstateAddress belongs. + /// Calculates and returns the shard number that this SubstateAddress belongs to. /// A shard is a fixed division of the 256-bit shard space. - /// If the substate version is not known, None is returned. + /// If the substate is global, returns `Some(Shard::global())` regardless of version. + /// For non-global substates, returns `None` if the version is not known. pub fn to_shard(&self, num_shards: NumPreshards) -> Option { if self.substate_id.is_global() { return Some(Shard::global()); diff --git a/crates/consensus_tests/fixtures/block.json b/crates/consensus_tests/fixtures/block.json index 405db0ccac..6413b66fed 100644 --- a/crates/consensus_tests/fixtures/block.json +++ b/crates/consensus_tests/fixtures/block.json @@ -30,8 +30,8 @@ "height": 65, "epoch": 5, "shard_group": { - "start": 0, - "end_inclusive": 255 + "start": 1, + "end_inclusive": 256 }, "signatures": [ { diff --git a/crates/consensus_tests/fixtures/block_with_dummies.json b/crates/consensus_tests/fixtures/block_with_dummies.json index d0d2c19d65..bfe40aefe3 100644 --- a/crates/consensus_tests/fixtures/block_with_dummies.json +++ b/crates/consensus_tests/fixtures/block_with_dummies.json @@ -30,7 +30,7 @@ "height": 0, "epoch": 5, "shard_group": { - "start": 0, + "start": 1, "end_inclusive": 127 }, "signatures": [], diff --git a/crates/consensus_tests/src/substate_store.rs b/crates/consensus_tests/src/substate_store.rs index a1c69d835b..b02f712bc5 100644 --- a/crates/consensus_tests/src/substate_store.rs +++ b/crates/consensus_tests/src/substate_store.rs @@ -215,7 +215,7 @@ fn add_substate(store: &TestStore, seed: u8, version: u32) -> VersionedSubstateI let id = new_substate_id(seed); let value = new_substate_value(seed); let mut batch = SubstateUpdateBatch::new(Epoch::zero()); - batch.add_transition(Shard::first(), 0, SubstateTransition::Up { + batch.with_transition(Shard::first(), 0).push(SubstateTransition::Up { id: id.clone(), version, substate_or_hash: value.into(), diff --git a/crates/consensus_tests/src/support/harness.rs b/crates/consensus_tests/src/support/harness.rs index 66d5972910..bfb4426ca3 100644 --- a/crates/consensus_tests/src/support/harness.rs +++ b/crates/consensus_tests/src/support/harness.rs @@ -179,9 +179,10 @@ impl Test { .iter() .map(|id| { let value = make_test_component(id.substate_id().as_component_address().unwrap().entity_id()); + let shard = id.to_shard(TEST_NUM_PRESHARDS); SubstateRecord::new(id.substate_id().clone(), id.version(), value, SubstateCreated { at_epoch: Epoch::zero(), - in_shard: Shard::first(), + in_shard: shard, at_state_version: 0, }) }) @@ -192,11 +193,9 @@ impl Test { for substate in &substates { let shard = substate.to_versioned_substate_id().to_shard(TEST_NUM_PRESHARDS); if v.shard_group.contains(&shard) { - batch.add_transition( - shard, - substate.created().at_state_version, - substate.clone().into_transition(), - ); + batch + .with_transition(shard, substate.created().at_state_version) + .push(substate.clone().into_transition()); } } diff --git a/crates/p2p/proto/rpc.proto b/crates/p2p/proto/rpc.proto index e4d1f0fd65..1f7e8f9984 100644 --- a/crates/p2p/proto/rpc.proto +++ b/crates/p2p/proto/rpc.proto @@ -222,7 +222,7 @@ message SyncStateResponse { uint64 state_version = 1; repeated SubstateUpdate updates = 2; bool has_more = 3; - tari.ootle.common.Epoch Epoch = 4; + tari.ootle.common.Epoch epoch = 4; } enum TemplateType { diff --git a/crates/rpc_state_sync/src/state_sync.rs b/crates/rpc_state_sync/src/state_sync.rs index bace074f18..82bd628c39 100644 --- a/crates/rpc_state_sync/src/state_sync.rs +++ b/crates/rpc_state_sync/src/state_sync.rs @@ -32,7 +32,6 @@ use tari_ootle_storage::{ SubstateCreatedProof, SubstateRecord, SubstateTransition, - SubstateTransitionData, SubstateUpdateBatch, SubstateUpdateProof, }, @@ -329,26 +328,20 @@ where TConsensusSpec: ConsensusSpec state_version: Version, updates: I, ) -> Result<(), StorageError> { - let batch_updates = IndexMap::from_iter([(shard, SubstateTransitionData { - state_version, - transitions: updates - .into_iter() - .map(|update| match update { - SubstateUpdateProof::Create(create) => SubstateTransition::Up { - id: create.substate.substate_id, - version: create.substate.version, - substate_or_hash: create.substate.value, - }, - SubstateUpdateProof::Destroy(destroy) => SubstateTransition::Down { - id: VersionedSubstateId::new(destroy.substate_id, destroy.version), - }, - }) - .collect(), - })]); - let batch = SubstateUpdateBatch { - epoch, - updates: batch_updates, - }; + let mut batch = SubstateUpdateBatch::new(epoch); + + batch + .with_transition(shard, state_version) + .extend(updates.into_iter().map(|update| match update { + SubstateUpdateProof::Create(create) => SubstateTransition::Up { + id: create.substate.substate_id, + version: create.substate.version, + substate_or_hash: create.substate.value, + }, + SubstateUpdateProof::Destroy(destroy) => SubstateTransition::Down { + id: VersionedSubstateId::new(destroy.substate_id, destroy.version), + }, + })); SubstateRecord::commit_batch(tx, batch)?; diff --git a/crates/state_store_rocksdb/src/column_families/substate.rs b/crates/state_store_rocksdb/src/column_families/substate.rs index 9244250818..8d0cc2761b 100644 --- a/crates/state_store_rocksdb/src/column_families/substate.rs +++ b/crates/state_store_rocksdb/src/column_families/substate.rs @@ -23,6 +23,7 @@ use serde::{Deserialize, Serialize}; use tari_engine_types::substate::SubstateId; use tari_ootle_common_types::{shard::Shard, Epoch, SubstateAddress}; +use tari_state_tree::Version; use crate::{ codecs::{ @@ -73,10 +74,10 @@ impl Cf for HeadIndex { pub struct UnprunedDownedValuesIndex; impl Cf for UnprunedDownedValuesIndex { - type Key = (Epoch, Shard, u64); - type KeyCodec = (EpochCodec, ShardCodec, NumberCodec); - type Value = ::Key; - type ValueCodec = ::KeyCodec; + type Key = (Epoch, Shard, Version); + type KeyCodec = (EpochCodec, ShardCodec, NumberCodec); + type Value = Vec; + type ValueCodec = DefaultCodec; fn name() -> &'static str { "substates_unpruned_idx" diff --git a/crates/state_store_rocksdb/src/writer.rs b/crates/state_store_rocksdb/src/writer.rs index e9b0ed91d9..617396070f 100644 --- a/crates/state_store_rocksdb/src/writer.rs +++ b/crates/state_store_rocksdb/src/writer.rs @@ -1201,66 +1201,79 @@ impl<'tx, TAddr: NodeAddressable + 'tx> StateStoreWriteTransaction for RocksDbSt let cf = db.cf(SubstateCf)?; let head_cf = db.cf(substate::HeadIndex)?; + let unpruned_cf = db.cf(substate::UnprunedDownedValuesIndex)?; + + for (shard, updates) in update_batch.updates { + for (state_version, updates) in updates { + let mut transitions = Vec::with_capacity(updates.len()); + let mut downed_substate_addresses = vec![]; + + for transition in updates { + match transition { + SubstateTransition::Up { + id, + version, + substate_or_hash, + } => { + let rec = SubstateRecord::new(id, version, substate_or_hash, SubstateCreated { + at_epoch: update_batch.epoch, + in_shard: shard, + at_state_version: state_version, + }); + + let address = rec.to_substate_address(); + cf.put(&address, &rec, OPERATION)?; + head_cf.put(rec.substate_id(), &SubstateHeadData { version, is_up: true }, OPERATION)?; + + transitions.push(StateTransitionRecordData { + substate_address: address, + transition: StateTransitionType::Up, + }); + }, + SubstateTransition::Down { id } => { + let address = id.to_substate_address(); + + let mut substate = cf.get(&address, OPERATION)?; + substate.destroyed = Some(SubstateDestroyed { + at_epoch: update_batch.epoch, + at_state_version: state_version, + }); + cf.put(&address, &substate, OPERATION)?; + head_cf.put( + &substate.substate_id, + &SubstateHeadData { + version: substate.version(), + is_up: false, + }, + OPERATION, + )?; + downed_substate_addresses.push(address); + + transitions.push(StateTransitionRecordData { + substate_address: address, + transition: StateTransitionType::Down, + }); + }, + } + } - for (shard, update) in update_batch.updates { - let mut transitions = Vec::with_capacity(update.transitions.len()); - - for transition in update.transitions { - match transition { - SubstateTransition::Up { - id, - version, - substate_or_hash, - } => { - let rec = SubstateRecord::new(id, version, substate_or_hash, SubstateCreated { - at_epoch: update_batch.epoch, - in_shard: shard, - at_state_version: update.state_version, - }); - - let address = rec.to_substate_address(); - cf.put(&address, &rec, OPERATION)?; - head_cf.put(rec.substate_id(), &SubstateHeadData { version, is_up: true }, OPERATION)?; - - transitions.push(StateTransitionRecordData { - substate_address: address, - transition: StateTransitionType::Up, - }); - }, - SubstateTransition::Down { id } => { - let address = id.to_substate_address(); - - let mut substate = cf.get(&address, OPERATION)?; - substate.destroyed = Some(SubstateDestroyed { - at_epoch: update_batch.epoch, - at_state_version: update.state_version, - }); - cf.put(&address, &substate, OPERATION)?; - db.cf(substate::HeadIndex)?.put( - &substate.substate_id, - &SubstateHeadData { - version: substate.version(), - is_up: false, - }, - OPERATION, - )?; - - transitions.push(StateTransitionRecordData { - substate_address: address, - transition: StateTransitionType::Down, - }); - }, + if !downed_substate_addresses.is_empty() { + unpruned_cf.put( + &(update_batch.epoch, shard, state_version), + &downed_substate_addresses, + OPERATION, + )?; } - } - let transition = StateTransitionModelDataV1 { - epoch: update_batch.epoch, - state_version: update.state_version, - transitions, - }; + let transition = StateTransitionModelDataV1 { + epoch: update_batch.epoch, + state_version, + transitions, + }; - db.cf(StateTransitionCf)? - .put(&(shard, update.state_version), &transition, OPERATION)?; + db.cf(StateTransitionCf)? + .put(&(shard, state_version), &transition, OPERATION)?; + } } Ok(()) @@ -1275,12 +1288,14 @@ impl<'tx, TAddr: NodeAddressable + 'tx> StateStoreWriteTransaction for RocksDbSt let substates_cf = db.cf(SubstateCf)?; let mut count = 0usize; for result in iter { - let (key, substate_addr) = result?; + let (key, addresses) = result?; - // TODO: store the actual values in a separate column family - let mut substate = substates_cf.get(&substate_addr, OPERATION)?; - substate.clear_substate_value(); - substates_cf.put(&substate_addr, &substate, OPERATION)?; + // TODO(perf): consider storing the actual values in a separate column family to avoid get/set + for substate_addr in addresses { + let mut substate = substates_cf.get(&substate_addr, OPERATION)?; + substate.clear_substate_value(); + substates_cf.put(&substate_addr, &substate, OPERATION)?; + } unpruned_index.delete(&key, OPERATION)?; count += 1; } diff --git a/crates/state_store_tests/src/helpers.rs b/crates/state_store_tests/src/helpers.rs index d6e28e3383..1a64a3f88d 100644 --- a/crates/state_store_tests/src/helpers.rs +++ b/crates/state_store_tests/src/helpers.rs @@ -169,23 +169,25 @@ pub fn create_substate_update_batch<'a, I: IntoIterator Vec { - let num_chunks = self.updates.len().div_ceil(size); + pub fn into_chunks(self, size: NonZeroUsize) -> Vec { + let num_chunks = self.updates.len().div_ceil(size.get()); let mut chunks = Vec::with_capacity(num_chunks); - loop { - if self.updates.len() < size { - chunks.push(Self { - epoch: self.epoch, - shard: self.shard, - state_version: self.state_version, - updates: self.updates, - }); - break; - } - - let chunk = self.updates.split_off(size); - + let mut updates = self.updates; + while !updates.is_empty() { + let take = updates.len().min(size.get()); + let chunk_updates: Vec<_> = updates.drain(..take).collect(); chunks.push(Self { epoch: self.epoch, shard: self.shard, state_version: self.state_version, - updates: chunk, + updates: chunk_updates, }); } chunks diff --git a/crates/storage/src/consensus_models/substate.rs b/crates/storage/src/consensus_models/substate.rs index 458368e86b..e993d35443 100644 --- a/crates/storage/src/consensus_models/substate.rs +++ b/crates/storage/src/consensus_models/substate.rs @@ -417,7 +417,7 @@ impl SubstateUpdateProof { } pub fn is_destroy(&self) -> bool { - matches!(self, Self::Destroy { .. }) + matches!(self, Self::Destroy(_)) } pub fn substate_id(&self) -> &SubstateId { diff --git a/crates/storage/src/consensus_models/substate_update_batch.rs b/crates/storage/src/consensus_models/substate_update_batch.rs index f2a6bc7f22..98538473e0 100644 --- a/crates/storage/src/consensus_models/substate_update_batch.rs +++ b/crates/storage/src/consensus_models/substate_update_batch.rs @@ -10,7 +10,7 @@ use crate::consensus_models::SubstateValueOrHash; pub struct SubstateUpdateBatch { pub epoch: Epoch, - pub updates: IndexMap, + pub updates: IndexMap>>, } impl SubstateUpdateBatch { @@ -21,28 +21,11 @@ impl SubstateUpdateBatch { } } - pub fn add_transition( - &mut self, - shard: Shard, - state_version: Version, - transition: SubstateTransition, - ) -> &mut Self { - self.updates - .entry(shard) - .or_insert_with(|| SubstateTransitionData { - state_version, - transitions: Vec::new(), - }) - .transitions - .push(transition); - self + pub fn with_transition(&mut self, shard: Shard, state_version: Version) -> &mut Vec { + self.updates.entry(shard).or_default().entry(state_version).or_default() } } -pub struct SubstateTransitionData { - pub state_version: Version, - pub transitions: Vec, -} pub enum SubstateTransition { Up { id: SubstateId, diff --git a/crates/storage/src/state_store/mod.rs b/crates/storage/src/state_store/mod.rs index 311579af23..746af73b51 100644 --- a/crates/storage/src/state_store/mod.rs +++ b/crates/storage/src/state_store/mod.rs @@ -280,7 +280,7 @@ pub trait StateStoreReadTransaction: Sized { &self, shard: Shard, state_version: Version, - include_values: bool + include_values: bool, ) -> Result; // -------------------------------- State Tree -------------------------------- // diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 48cafc6dc4..53bfe9fa1d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -48,8 +48,8 @@ importers: specifier: ^5.81.2 version: 5.81.2(react@19.1.0) '@tanstack/react-query-devtools': - specifier: ^5.81.2 - version: 5.81.2(@tanstack/react-query@5.81.2(react@19.1.0))(react@19.1.0) + specifier: ^5.84.2 + version: 5.85.5(@tanstack/react-query@5.81.2(react@19.1.0))(react@19.1.0) '@tari-project/typescript-bindings': specifier: link:../../../bindings version: link:../../../bindings @@ -1918,8 +1918,8 @@ packages: '@tanstack/query-core@5.81.2': resolution: {integrity: sha512-QLYkPdrudoMATDFa3MiLEwRhNnAlzHWDf0LKaXUqJd0/+QxN8uTPi7bahRlxoAyH0UbLMBdeDbYzWALj7THOtw==} - '@tanstack/query-devtools@5.81.2': - resolution: {integrity: sha512-jCeJcDCwKfoyyBXjXe9+Lo8aTkavygHHsUHAlxQKKaDeyT0qyQNLKl7+UyqYH2dDF6UN/14873IPBHchcsU+Zg==} + '@tanstack/query-devtools@5.84.0': + resolution: {integrity: sha512-fbF3n+z1rqhvd9EoGp5knHkv3p5B2Zml1yNRjh7sNXklngYI5RVIWUrUjZ1RIcEoscarUb0+bOvIs5x9dwzOXQ==} '@tanstack/react-query-devtools@4.36.1': resolution: {integrity: sha512-WYku83CKP3OevnYSG8Y/QO9g0rT75v1om5IvcWUwiUZJ4LanYGLVCZ8TdFG5jfsq4Ej/lu2wwDAULEUnRIMBSw==} @@ -1928,10 +1928,10 @@ packages: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 - '@tanstack/react-query-devtools@5.81.2': - resolution: {integrity: sha512-TX0OQ4cbgX6z2uN8c9x0QUNbyePGyUGdcgrGnV6TYEJc7KPT8PqeASuzoA5NGw1CiMGvyFAkIGA2KipvhM9d1g==} + '@tanstack/react-query-devtools@5.85.5': + resolution: {integrity: sha512-6Ol6Q+LxrCZlQR4NoI5181r+ptTwnlPG2t7H9Sp3klxTBhYGunONqcgBn2YKRPsaKiYM8pItpKMdMXMEINntMQ==} peerDependencies: - '@tanstack/react-query': ^5.81.2 + '@tanstack/react-query': ^5.85.5 react: ^18 || ^19 '@tanstack/react-query@4.36.1': @@ -6634,7 +6634,7 @@ snapshots: '@tanstack/query-core@5.81.2': {} - '@tanstack/query-devtools@5.81.2': {} + '@tanstack/query-devtools@5.84.0': {} '@tanstack/react-query-devtools@4.36.1(@tanstack/react-query@4.36.1(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': dependencies: @@ -6654,9 +6654,9 @@ snapshots: superjson: 1.13.3 use-sync-external-store: 1.5.0(react@19.1.0) - '@tanstack/react-query-devtools@5.81.2(@tanstack/react-query@5.81.2(react@19.1.0))(react@19.1.0)': + '@tanstack/react-query-devtools@5.85.5(@tanstack/react-query@5.81.2(react@19.1.0))(react@19.1.0)': dependencies: - '@tanstack/query-devtools': 5.81.2 + '@tanstack/query-devtools': 5.84.0 '@tanstack/react-query': 5.81.2(react@19.1.0) react: 19.1.0 diff --git a/utilities/db_inspector/src/webserver/handlers/state_transitions.rs b/utilities/db_inspector/src/webserver/handlers/state_transitions.rs index 2eebdc5346..3c30fb3e6d 100644 --- a/utilities/db_inspector/src/webserver/handlers/state_transitions.rs +++ b/utilities/db_inspector/src/webserver/handlers/state_transitions.rs @@ -52,26 +52,38 @@ pub async fn list( cf.range_iterator(ordering, empty.as_slice()..) }; - let page_size = req.limit.unwrap_or(1_000); - let skip = req.page.unwrap_or(0) * page_size; - for result in iter.skip(skip).take(page_size) { + let row_limit = req.limit.unwrap_or(1_000); + let row_skip = req.page.unwrap_or(0).saturating_mul(row_limit); + let mut skipped = 0usize; + let mut emitted = 0usize; + let mut count = 0usize; + for result in iter { let ((shard, state_version), data) = result?; let encoded_key = cf.encode_key(&(shard, state_version)); + let key_hex = hex::encode(&encoded_key); for (i, transition) in data.transitions.into_iter().enumerate() { - let substate = substate_cf.get(&transition.substate_address, OPERATION).optional()?; - table.add_row(json!({ - "id": hex::encode(&encoded_key) + &format!("-{}", i), - "epoch": data.epoch, - "shard": shard, - "state_version": state_version, - "substate_id": substate.as_ref().map(|s| s.substate_id()), - "version": substate.as_ref().map(|s| s.version()), - "transition": transition.transition, - })); + if skipped < row_skip { + skipped += 1; + continue; + } + if emitted < row_limit { + let substate = substate_cf.get(&transition.substate_address, OPERATION).optional()?; + table.add_row(json!({ + "id": format!("{}-{}", key_hex, i), + "epoch": data.epoch, + "shard": shard, + "state_version": state_version, + "substate_id": substate.as_ref().map(|s| s.substate_id()), + "version": substate.as_ref().map(|s| s.version()), + "transition": transition.transition, + })); + emitted += 1; + } + + count += 1; } } - let total = cf.count(OPERATION)?; - table.set_total_entries(total); + table.set_total_entries(count); Ok(Json(table)) } From a00cd8266d14ccd699ddb5df0d204bc5c62d8ca3 Mon Sep 17 00:00:00 2001 From: Stan Bondi Date: Tue, 19 Aug 2025 15:59:41 +0400 Subject: [PATCH 3/5] slight improvement to stealth/confidential resource builder api --- crates/engine/tests/templates/stealth/src/lib.rs | 8 +------- crates/rpc_state_sync/src/state_sync.rs | 1 - .../src/resource/builder/confidential.rs | 12 ++++++++++-- .../template_lib/src/resource/builder/stealth.rs | 14 +++++++++++--- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/crates/engine/tests/templates/stealth/src/lib.rs b/crates/engine/tests/templates/stealth/src/lib.rs index 145abbf0db..39b04a2e2f 100644 --- a/crates/engine/tests/templates/stealth/src/lib.rs +++ b/crates/engine/tests/templates/stealth/src/lib.rs @@ -20,13 +20,7 @@ mod template { ) -> Component { let bucket = ResourceBuilder::stealth() .mintable(rule!(allow_all)) - .then(|builder| { - if let Some(key) = view_key { - builder.with_view_key(key) - } else { - builder - } - }) + .with_view_key_opt(view_key) .initial_supply(initial_supply); let resource_address = bucket.resource_address(); diff --git a/crates/rpc_state_sync/src/state_sync.rs b/crates/rpc_state_sync/src/state_sync.rs index 82bd628c39..830f11150c 100644 --- a/crates/rpc_state_sync/src/state_sync.rs +++ b/crates/rpc_state_sync/src/state_sync.rs @@ -5,7 +5,6 @@ use std::{collections::HashMap, time::Instant}; use anyhow::anyhow; use futures::StreamExt; -use indexmap::IndexMap; use log::*; use tari_consensus::{ hotstuff::substate_store::{ShardScopedTreeStoreReader, ShardScopedTreeStoreWriter}, diff --git a/crates/template_lib/src/resource/builder/confidential.rs b/crates/template_lib/src/resource/builder/confidential.rs index 6798564c98..46cc1ee5a5 100644 --- a/crates/template_lib/src/resource/builder/confidential.rs +++ b/crates/template_lib/src/resource/builder/confidential.rs @@ -82,8 +82,16 @@ impl ConfidentialResourceBuilder { /// Specify a view key for the confidential resource. This allows anyone with the secret key to uncover the balance /// of commitments generated for the resource. /// NOTE: it is not currently possible to change the view key after the resource is created. - pub fn with_view_key(mut self, view_key: RistrettoPublicKeyBytes) -> Self { - self.view_key = Some(view_key); + /// Equivalent to calling `with_view_key_opt(Some(view_key))`. + pub fn with_view_key(self, view_key: RistrettoPublicKeyBytes) -> Self { + self.with_view_key_opt(Some(view_key)) + } + + /// Optionally, specify a view key for the confidential resource. This allows anyone with the secret key to uncover + /// the balance of commitments generated for the resource. + /// NOTE: it is not currently possible to change the view key after the resource is created. + pub fn with_view_key_opt(mut self, view_key: Option) -> Self { + self.view_key = view_key; self } diff --git a/crates/template_lib/src/resource/builder/stealth.rs b/crates/template_lib/src/resource/builder/stealth.rs index 9195125314..83d0e12ad3 100644 --- a/crates/template_lib/src/resource/builder/stealth.rs +++ b/crates/template_lib/src/resource/builder/stealth.rs @@ -82,11 +82,19 @@ impl StealthResourceBuilder { self } - /// Specify a view key for the confidential resource. This allows anyone with the secret key to uncover the balance + /// Specify a view key for the stealth resource. This allows anyone with the secret key to uncover the balance /// of commitments generated for the resource. /// NOTE: it is not currently possible to change the view key after the resource is created. - pub fn with_view_key(mut self, view_key: RistrettoPublicKeyBytes) -> Self { - self.view_key = Some(view_key); + /// Equivalent to calling `with_view_key_opt(Some(view_key))`. + pub fn with_view_key(self, view_key: RistrettoPublicKeyBytes) -> Self { + self.with_view_key_opt(Some(view_key)) + } + + /// Optionally, specify a view key for the stealth resource. This allows anyone with the secret key to uncover the + /// balance of commitments generated for the resource. + /// NOTE: it is not currently possible to change the view key after the resource is created. + pub fn with_view_key_opt(mut self, view_key: Option) -> Self { + self.view_key = view_key; self } From 86c6371adb5ff38b278f1cf583a4128a0cd681b3 Mon Sep 17 00:00:00 2001 From: Stan Bondi Date: Tue, 19 Aug 2025 16:05:28 +0400 Subject: [PATCH 4/5] review comments --- Cargo.lock | 1 - crates/consensus_tests/src/state_tree.rs | 2 +- crates/rpc_state_sync/Cargo.toml | 1 - crates/state_store_rocksdb/src/reader.rs | 2 +- crates/state_store_tests/src/state_transitions.rs | 4 +++- crates/storage/src/consensus_models/state_transition.rs | 2 +- crates/storage/src/state_store/mod.rs | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4df70eb20d..820a734bc9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11778,7 +11778,6 @@ version = "0.11.2" dependencies = [ "anyhow", "futures 0.3.31", - "indexmap 2.9.0", "log", "tari_consensus", "tari_consensus_types", diff --git a/crates/consensus_tests/src/state_tree.rs b/crates/consensus_tests/src/state_tree.rs index 6313985000..20fbac0975 100644 --- a/crates/consensus_tests/src/state_tree.rs +++ b/crates/consensus_tests/src/state_tree.rs @@ -66,7 +66,7 @@ async fn check_state_transitions() { let mut all_transitions = vec![]; let mut next_state_version = 1; while let Some(transitions) = tx - .state_transitions_get_after(shard, next_state_version, false) + .state_transitions_get_starting_at(shard, next_state_version, false) .optional() .unwrap() { diff --git a/crates/rpc_state_sync/Cargo.toml b/crates/rpc_state_sync/Cargo.toml index 531770efc7..cd1089285d 100644 --- a/crates/rpc_state_sync/Cargo.toml +++ b/crates/rpc_state_sync/Cargo.toml @@ -22,5 +22,4 @@ tari_validator_node_rpc = { workspace = true } anyhow = { workspace = true } futures = { workspace = true } log = { workspace = true } -indexmap = { workspace = true } thiserror = { workspace = true } diff --git a/crates/state_store_rocksdb/src/reader.rs b/crates/state_store_rocksdb/src/reader.rs index 3addfa4124..9d2718e719 100644 --- a/crates/state_store_rocksdb/src/reader.rs +++ b/crates/state_store_rocksdb/src/reader.rs @@ -1589,7 +1589,7 @@ impl<'tx, TAddr: NodeAddressable + Serialize + DeserializeOwned + 'tx> StateStor Ok(diffs) } - fn state_transitions_get_after( + fn state_transitions_get_starting_at( &self, req_shard: Shard, state_version: Version, diff --git a/crates/state_store_tests/src/state_transitions.rs b/crates/state_store_tests/src/state_transitions.rs index a76680738e..fa72e52fa5 100644 --- a/crates/state_store_tests/src/state_transitions.rs +++ b/crates/state_store_tests/src/state_transitions.rs @@ -63,7 +63,9 @@ fn operations(db: impl StateStore) { for (state_version, (num_substates, shards)) in &shards { for shard in shards { - let transitions = tx.state_transitions_get_after(*shard, *state_version, false).unwrap(); + let transitions = tx + .state_transitions_get_starting_at(*shard, *state_version, false) + .unwrap(); assert_eq!(transitions.epoch, EPOCH); assert_eq!(transitions.state_version, *state_version); assert_eq!(transitions.shard, *shard); diff --git a/crates/storage/src/consensus_models/state_transition.rs b/crates/storage/src/consensus_models/state_transition.rs index 22ca16c073..d425474548 100644 --- a/crates/storage/src/consensus_models/state_transition.rs +++ b/crates/storage/src/consensus_models/state_transition.rs @@ -45,6 +45,6 @@ impl StateTransition { state_version: Version, include_values: bool, ) -> Result { - tx.state_transitions_get_after(shard, state_version, include_values) + tx.state_transitions_get_starting_at(shard, state_version, include_values) } } diff --git a/crates/storage/src/state_store/mod.rs b/crates/storage/src/state_store/mod.rs index 746af73b51..4a98d9c15f 100644 --- a/crates/storage/src/state_store/mod.rs +++ b/crates/storage/src/state_store/mod.rs @@ -276,7 +276,7 @@ pub trait StateStoreReadTransaction: Sized { ) -> Result>, StorageError>; // -------------------------------- State transitions -------------------------------- // - fn state_transitions_get_after( + fn state_transitions_get_starting_at( &self, shard: Shard, state_version: Version, From fe3bf6f926c88cd49d0f92a7b7f9e312761d3889 Mon Sep 17 00:00:00 2001 From: Stan Bondi Date: Tue, 19 Aug 2025 16:13:17 +0400 Subject: [PATCH 5/5] enforce monotonic state version --- crates/rpc_state_sync/src/state_sync.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/rpc_state_sync/src/state_sync.rs b/crates/rpc_state_sync/src/state_sync.rs index 830f11150c..35f230eee2 100644 --- a/crates/rpc_state_sync/src/state_sync.rs +++ b/crates/rpc_state_sync/src/state_sync.rs @@ -147,6 +147,7 @@ where TConsensusSpec: ConsensusSpec // We start at 1 because bootstrapped state is at 0 let start_state_version = maybe_persisted_state_version.unwrap_or(1); + let mut last_state_version = start_state_version; info!( target: LOG_TARGET, "🛜Syncing from v{start_state_version}", @@ -197,12 +198,22 @@ where TConsensusSpec: ConsensusSpec ))); } + let state_version = msg.state_version; + if state_version < last_state_version { + return Err(RpcStateSyncError::InvalidResponse(anyhow!( + "Received state version {} that is less than the last state version {}.", + state_version, + last_state_version + ))); + } + + last_state_version = state_version; + self.stats.total_transitions += msg.updates.len() as u64; tree_changes.reserve_exact(msg.updates.len()); updates.reserve_exact(msg.updates.len()); - let state_version = msg.state_version; let updates_for_state_version = msg .updates .into_iter()