diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d4127975f..0fc6a85c84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ - Reduced optimized benchmark build time by relaxing forced inlining in processor execution helpers ([#3292](https://github.com/0xMiden/miden-vm/pull/3292)). - Added no-op handlers for readonly debugger events to `CoreLibrary::handlers`, so hosts that load the core library can execute programs emitting those events without registering no-op handlers manually ([#3305](https://github.com/0xMiden/miden-vm/pull/3305)). +- Added trusted sparse MAST forest serialization for trace replay payloads ([#3313](https://github.com/0xMiden/miden-vm/pull/3313)). ## v0.24.0 (2026-06-24) diff --git a/core/src/mast/mod.rs b/core/src/mast/mod.rs index 582f572776..abd401d3d1 100644 --- a/core/src/mast/mod.rs +++ b/core/src/mast/mod.rs @@ -54,7 +54,7 @@ use proptest::prelude::*; use serde::{Deserialize, Serialize}; #[cfg(feature = "serde")] -use crate::serde::{Deserializable, SliceReader}; +use crate::serde::SliceReader; mod node; #[cfg(any(test, feature = "arbitrary"))] @@ -71,7 +71,7 @@ use crate::{ Felt, Word, advice::AdviceMap, crypto::hash::Poseidon2, - serde::{ByteWriter, DeserializationError, Serializable}, + serde::{ByteWriter, Deserializable, DeserializationError, Serializable}, utils::{DenseIdMap, Idx, IndexVec, hash_string_to_word}, }; @@ -1242,6 +1242,18 @@ impl Serializable for MastNodeId { } } +impl Deserializable for MastNodeId { + fn read_from( + source: &mut R, + ) -> Result { + Ok(Self(::read_from(source)?)) + } + + fn min_serialized_size() -> usize { + ::min_serialized_size() + } +} + impl fmt::Display for MastNodeId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "MastNodeId({})", self.0) diff --git a/core/src/mast/serialization/mod.rs b/core/src/mast/serialization/mod.rs index a50d765d2b..972ca9b895 100644 --- a/core/src/mast/serialization/mod.rs +++ b/core/src/mast/serialization/mod.rs @@ -76,9 +76,13 @@ //! same contiguous array on the wire. //! //! Public entry points adopt these policies: -//! - [`MastForest::read_from_bytes`]: trusted execution payload, no hashless support. +//! - [`MastForest::read_from_bytes`]: trusted dense execution payload, no hashless support. //! - [`MastForestWireView::new`]: trusted wire-backed cache access; rejects hashless and legacy //! debug-bearing payloads. +//! - [`crate::mast::SparseMastForest::read_from_bytes`]: separate trusted sparse replay payloads +//! for serialized trace-generation inputs. Sparse payloads preserve the sparse node and digest +//! maps produced by tracing; they do not share the dense `MastForest` wire format and are not an +//! untrusted validation boundary. //! - [`crate::mast::UntrustedMastForest::read_from_bytes`] / //! [`crate::mast::UntrustedMastForest::read_from_bytes_with_options`]: untrusted parsing plus //! later validation before use. @@ -112,6 +116,8 @@ mod layout; pub(super) use layout::ForestLayout; use layout::{OffsetTrackingReader, TrackingReader, WireFlags, read_header_and_scan_layout}; +mod sparse; + mod resolved; use resolved::{ResolvedSerializedForest, basic_block_offset_for_node_index}; diff --git a/core/src/mast/serialization/sparse.rs b/core/src/mast/serialization/sparse.rs new file mode 100644 index 0000000000..cfa0f9c586 --- /dev/null +++ b/core/src/mast/serialization/sparse.rs @@ -0,0 +1,338 @@ +use alloc::{collections::BTreeMap, format, string::ToString, vec::Vec}; + +use super::{ + TRUSTED_BYTE_READ_BUDGET_MULTIPLIER, + basic_blocks::{BasicBlockDataBuilder, BasicBlockDataDecoder}, +}; +use crate::{ + Word, + advice::AdviceMap, + mast::{ + BasicBlockNodeBuilder, CallNodeBuilder, DynNodeBuilder, ExternalNodeBuilder, + JoinNodeBuilder, LoopNodeBuilder, MastForest, MastForestContributor, MastNode, MastNodeExt, + MastNodeId, SparseMastForest, SplitNodeBuilder, + }, + serde::{ + BudgetedReader, ByteReader, ByteWriter, Deserializable, DeserializationError, Serializable, + SliceReader, read_bounded_len, + }, +}; + +const SPARSE_BLOCK: u8 = 0; +const SPARSE_JOIN: u8 = 1; +const SPARSE_SPLIT: u8 = 2; +const SPARSE_LOOP: u8 = 3; +const SPARSE_CALL: u8 = 4; +const SPARSE_SYSCALL: u8 = 5; +const SPARSE_DYN: u8 = 6; +const SPARSE_DYNCALL: u8 = 7; +const SPARSE_EXTERNAL: u8 = 8; + +// WRITER +// ================================================================================================ + +/// Writes trusted sparse trace replay data. +/// +/// This format preserves the sparse maps produced by execution tracing. It does not prove that +/// this sparse view is a subset of a committed [`MastForest`], and it does not share the dense +/// [`MastForest`] wire format. Callers must only read these bytes from a trusted producer, or after +/// an outer transport/authentication layer has accepted them. +fn write_sparse_into(forest: &SparseMastForest, target: &mut W) { + write_node_ids(forest.procedure_roots(), target); + write_sparse_nodes(forest.nodes(), target); + write_digest_entries(forest.digest_entries(), target); + forest.advice_map().write_into(target); +} + +fn write_node_ids(ids: &[MastNodeId], target: &mut W) { + target.write_usize(ids.len()); + for id in ids { + id.write_into(target); + } +} + +fn write_sparse_nodes(nodes: &BTreeMap, target: &mut W) { + target.write_usize(nodes.len()); + for (&id, node) in nodes { + id.write_into(target); + write_sparse_node(node, target); + } +} + +fn write_digest_entries(digests: &BTreeMap, target: &mut W) { + target.write_usize(digests.len()); + for (&id, &digest) in digests { + id.write_into(target); + digest.write_into(target); + } +} + +fn write_sparse_node(node: &MastNode, target: &mut W) { + match node { + MastNode::Block(block) => { + target.write_u8(SPARSE_BLOCK); + node.digest().write_into(target); + + let mut basic_block_data = BasicBlockDataBuilder::new(); + let ops_offset = basic_block_data.encode_basic_block(block); + debug_assert_eq!(ops_offset, 0); + let basic_block_data = basic_block_data.finalize(); + target.write_usize(basic_block_data.len()); + target.write_bytes(&basic_block_data); + }, + MastNode::Join(join) => { + target.write_u8(SPARSE_JOIN); + node.digest().write_into(target); + join.first().write_into(target); + join.second().write_into(target); + }, + MastNode::Split(split) => { + target.write_u8(SPARSE_SPLIT); + node.digest().write_into(target); + split.on_true().write_into(target); + split.on_false().write_into(target); + }, + MastNode::Loop(loop_node) => { + target.write_u8(SPARSE_LOOP); + node.digest().write_into(target); + loop_node.body().write_into(target); + }, + MastNode::Call(call) => { + target.write_u8(if call.is_syscall() { SPARSE_SYSCALL } else { SPARSE_CALL }); + node.digest().write_into(target); + call.callee().write_into(target); + }, + MastNode::Dyn(dyn_node) => { + target.write_u8(if dyn_node.is_dyncall() { + SPARSE_DYNCALL + } else { + SPARSE_DYN + }); + node.digest().write_into(target); + }, + MastNode::External(_) => { + target.write_u8(SPARSE_EXTERNAL); + node.digest().write_into(target); + }, + } +} + +// TRAIT IMPLS +// ================================================================================================ + +impl Serializable for SparseMastForest { + fn write_into(&self, target: &mut W) { + write_sparse_into(self, target); + } +} + +impl Deserializable for SparseMastForest { + fn read_from(source: &mut R) -> Result { + read_sparse_from(source) + } + + fn min_serialized_size() -> usize { + usize::min_serialized_size() + } + + /// Reads one trusted sparse replay payload and rejects trailing bytes. + /// + /// This is not an untrusted input format. The reader performs cheap structural checks, but a + /// producer controls collection lengths and can drive allocation. Callers must only read these + /// bytes from a trusted producer, or after an outer transport/authentication layer has accepted + /// them. + fn read_from_bytes(bytes: &[u8]) -> Result { + let budget = bytes.len().saturating_mul(TRUSTED_BYTE_READ_BUDGET_MULTIPLIER); + let mut reader = BudgetedReader::new(SliceReader::new(bytes), budget); + let forest = read_sparse_from(&mut reader)?; + if reader.has_more_bytes() { + return Err(DeserializationError::InvalidValue( + "extra bytes after SparseMastForest payload".to_string(), + )); + } + Ok(forest) + } +} + +// READER +// ================================================================================================ + +fn read_sparse_from( + source: &mut R, +) -> Result { + let roots = read_node_ids(source, "procedure root")?; + let nodes = read_sparse_nodes(source)?; + let digest_entries = read_digest_entries(source)?; + let advice_map = read_empty_advice_map(source)?; + + SparseMastForest::from_serialized_parts(nodes, digest_entries, roots, advice_map) +} + +fn read_empty_advice_map(source: &mut R) -> Result { + let count = source.read_usize()?; + if count != 0 { + return Err(DeserializationError::InvalidValue( + "sparse MAST replay payload must not carry advice map entries".to_string(), + )); + } + + Ok(AdviceMap::default()) +} + +fn read_node_ids( + source: &mut R, + label: &str, +) -> Result, DeserializationError> { + let count = read_bounded_count(source, u32::min_serialized_size(), label)?; + let mut ids = Vec::with_capacity(count); + for _ in 0..count { + ids.push(read_node_id(source, label)?); + } + Ok(ids) +} + +fn read_sparse_nodes( + source: &mut R, +) -> Result, DeserializationError> { + let count = read_bounded_count(source, sparse_node_min_size(), "full node count")?; + let mut nodes = Vec::with_capacity(count); + let mut previous_id = None; + + for _ in 0..count { + let id = read_node_id(source, "full node")?; + validate_strictly_increasing_id(previous_id, id, "full node")?; + let node = read_sparse_node(source, id)?; + nodes.push((id, node)); + previous_id = Some(id); + } + + Ok(nodes) +} + +fn read_digest_entries( + source: &mut R, +) -> Result, DeserializationError> { + let count = + read_bounded_count(source, sparse_digest_entry_min_size(), "digest-only node count")?; + let mut digests = Vec::with_capacity(count); + let mut previous_id = None; + + for _ in 0..count { + let id = read_node_id(source, "digest-only node")?; + validate_strictly_increasing_id(previous_id, id, "digest-only node")?; + let digest = Word::read_from(source)?; + digests.push((id, digest)); + previous_id = Some(id); + } + + Ok(digests) +} + +fn read_sparse_node( + source: &mut R, + node_id: MastNodeId, +) -> Result { + let tag = source.read_u8()?; + let digest = Word::read_from(source)?; + + let result = match tag { + SPARSE_BLOCK => { + let len = read_bounded_count(source, 1, "basic block data length")?; + let data = source.read_vec(len)?; + let decoder = BasicBlockDataDecoder::new(&data); + let op_batches = decoder.decode_operations(0)?; + BasicBlockNodeBuilder::from_op_batches(op_batches, digest) + .build() + .map(Into::into) + }, + SPARSE_JOIN => { + let first = read_node_id(source, "join first child")?; + let second = read_node_id(source, "join second child")?; + JoinNodeBuilder::new([first, second]) + .with_digest(digest) + .build_linked() + .map(Into::into) + }, + SPARSE_SPLIT => { + let on_true = read_node_id(source, "split true child")?; + let on_false = read_node_id(source, "split false child")?; + SplitNodeBuilder::new([on_true, on_false]) + .with_digest(digest) + .build_linked() + .map(Into::into) + }, + SPARSE_LOOP => { + let body = read_node_id(source, "loop body")?; + LoopNodeBuilder::new(body).with_digest(digest).build_linked().map(Into::into) + }, + SPARSE_CALL | SPARSE_SYSCALL => { + let callee = read_node_id(source, "call callee")?; + let builder = if tag == SPARSE_SYSCALL { + CallNodeBuilder::new_syscall(callee) + } else { + CallNodeBuilder::new(callee) + }; + builder.with_digest(digest).build_linked().map(Into::into) + }, + SPARSE_DYN | SPARSE_DYNCALL => { + let builder = if tag == SPARSE_DYNCALL { + DynNodeBuilder::new_dyncall() + } else { + DynNodeBuilder::new_dyn() + }; + Ok(builder.with_digest(digest).build().into()) + }, + SPARSE_EXTERNAL => Ok(ExternalNodeBuilder::new(digest).build().into()), + _ => { + return Err(DeserializationError::InvalidValue(format!( + "invalid sparse MAST node tag {tag}" + ))); + }, + }; + + result.map_err(|err| { + DeserializationError::InvalidValue(format!( + "failed to build sparse MAST node {}: {}", + node_id.0, err + )) + }) +} + +fn read_bounded_count( + source: &mut R, + element_size: usize, + label: &str, +) -> Result { + read_bounded_len(source, label, element_size) +} + +fn sparse_node_min_size() -> usize { + u32::min_serialized_size() + u8::min_serialized_size() + Word::min_serialized_size() +} + +fn sparse_digest_entry_min_size() -> usize { + u32::min_serialized_size() + Word::min_serialized_size() +} + +fn read_node_id( + source: &mut R, + label: &str, +) -> Result { + let raw = u32::read_from(source)?; + MastNodeId::from_u32_with_node_count(raw, MastForest::MAX_NODES).map_err(|err| { + DeserializationError::InvalidValue(format!("invalid {label} id {raw}: {err}")) + }) +} + +fn validate_strictly_increasing_id( + previous: Option, + current: MastNodeId, + label: &str, +) -> Result<(), DeserializationError> { + if previous.is_some_and(|previous| previous >= current) { + return Err(DeserializationError::InvalidValue(format!( + "{label} ids must be strictly increasing" + ))); + } + Ok(()) +} diff --git a/core/src/mast/serialization/tests.rs b/core/src/mast/serialization/tests.rs index 39dd5de02d..e584bc0d19 100644 --- a/core/src/mast/serialization/tests.rs +++ b/core/src/mast/serialization/tests.rs @@ -1,7 +1,7 @@ use core::assert_matches; use std::{ string::{String, ToString}, - sync::{Mutex, Once}, + sync::{Arc, Mutex, Once}, }; use super::*; @@ -9,10 +9,10 @@ use crate::{ Felt, Word, chiplets::hasher, mast::{ - BasicBlockNodeBuilder, CallNodeBuilder, DynNodeBuilder, ExternalNodeBuilder, - JoinNodeBuilder, LoopNodeBuilder, MastForestError, MastForestView, MastNodeExt, MastNodeId, - OP_BATCH_SIZE, OpBatch, SplitNodeBuilder, UntrustedMastForest, - UntrustedMastForestReadOptions, + BasicBlockNodeBuilder, CallNodeBuilder, DynNodeBuilder, ExecutableMastForest, + ExternalNodeBuilder, JoinNodeBuilder, LoopNodeBuilder, MastForestError, MastForestView, + MastNodeExt, MastNodeId, OP_BATCH_SIZE, OpBatch, SparseMastForest, SparseMastForestBuilder, + SplitNodeBuilder, UntrustedMastForest, UntrustedMastForestReadOptions, VisitKind, }, operations::Operation, serde::{ByteReader, Deserializable, DeserializationError, Serializable, SliceReader}, @@ -746,6 +746,429 @@ fn test_untrusted_hashless_keeps_external_digests_by_prefix() { assert_eq!(restored[MastNodeId::new_unchecked(1)].digest(), external_high); } +fn sparse_split_fixture() -> (Arc, SparseMastForest, MastNodeId, MastNodeId, MastNodeId) +{ + let mut forest = MastForest::new(); + let true_branch = BasicBlockNodeBuilder::new(vec![Operation::Add]) + .add_to_forest(&mut forest) + .unwrap(); + let false_branch = BasicBlockNodeBuilder::new(vec![Operation::Mul]) + .add_to_forest(&mut forest) + .unwrap(); + let root = SplitNodeBuilder::new([true_branch, false_branch]) + .add_to_forest(&mut forest) + .unwrap(); + forest.make_root(root); + + let forest = Arc::new(forest); + let mut builder = SparseMastForestBuilder::new(Arc::clone(&forest)); + builder.record_visit(true_branch, VisitKind::FullVisit); + builder.record_visit(false_branch, VisitKind::DigestOnly); + builder.record_visit(root, VisitKind::FullVisit); + let sparse = builder.finalize(); + + (forest, sparse, true_branch, false_branch, root) +} + +#[test] +fn sparse_mast_round_trip_preserves_sparse_replay_ids() { + let (source, sparse, true_branch, false_branch, root) = sparse_split_fixture(); + + let bytes = sparse.to_bytes(); + let restored = SparseMastForest::read_from_bytes(&bytes).unwrap(); + + assert_eq!(restored.num_nodes(), source.num_nodes() as usize); + assert_eq!(restored.procedure_roots(), &[root]); + assert_eq!( + restored.get_node_by_id(true_branch).unwrap().digest(), + source[true_branch].digest() + ); + assert_eq!(restored.get_node_by_id(root).unwrap().digest(), source[root].digest()); + assert!(restored.get_node_by_id(false_branch).is_none()); + assert_eq!(restored.get_digest_by_id(true_branch), Some(source[true_branch].digest())); + assert_eq!(restored.get_digest_by_id(false_branch), Some(source[false_branch].digest())); + assert_eq!(restored.get_digest_by_id(root), Some(source[root].digest())); +} + +#[test] +fn sparse_mast_round_trip_preserves_external_full_node() { + let mut forest = MastForest::new(); + let unvisited = BasicBlockNodeBuilder::new(vec![Operation::Add]) + .add_to_forest(&mut forest) + .unwrap(); + let external_digest = Word::new([ + Felt::new_unchecked(17), + Felt::new_unchecked(18), + Felt::new_unchecked(19), + Felt::new_unchecked(20), + ]); + let external = ExternalNodeBuilder::new(external_digest).add_to_forest(&mut forest).unwrap(); + forest.make_root(external); + + let forest = Arc::new(forest); + let mut builder = SparseMastForestBuilder::new(Arc::clone(&forest)); + builder.record_visit(external, VisitKind::FullVisit); + let sparse = builder.finalize(); + + let restored = SparseMastForest::read_from_bytes(&sparse.to_bytes()).unwrap(); + + assert_eq!(restored.num_nodes(), forest.num_nodes() as usize); + assert_eq!(restored.procedure_roots(), &[external]); + assert_eq!(restored.get_node_by_id(external).unwrap().digest(), external_digest); + assert_eq!(restored.get_digest_by_id(external), Some(external_digest)); + assert!(restored.get_node_by_id(unvisited).is_none()); + assert_eq!(restored.get_digest_by_id(unvisited), None); +} + +#[test] +fn sparse_mast_round_trip_writes_map_sections_in_node_id_order() { + let mut forest = MastForest::new(); + let first = BasicBlockNodeBuilder::new(vec![Operation::Add]) + .add_to_forest(&mut forest) + .unwrap(); + let second = BasicBlockNodeBuilder::new(vec![Operation::Mul]) + .add_to_forest(&mut forest) + .unwrap(); + let third = BasicBlockNodeBuilder::new(vec![Operation::Drop]) + .add_to_forest(&mut forest) + .unwrap(); + forest.make_root(first); + + let forest = Arc::new(forest); + let mut builder = SparseMastForestBuilder::new(Arc::clone(&forest)); + builder.record_visit(third, VisitKind::FullVisit); + builder.record_visit(first, VisitKind::FullVisit); + builder.record_visit(second, VisitKind::DigestOnly); + let sparse = builder.finalize(); + + let (full_ids, digest_ids) = sparse_payload_ids(&sparse.to_bytes()); + + assert_eq!(full_ids, vec![first, third]); + assert_eq!(digest_ids, vec![second]); +} + +#[test] +fn sparse_mast_drops_source_advice_map() { + let mut source = MastForest::new(); + let root = BasicBlockNodeBuilder::new(vec![Operation::Add]) + .add_to_forest(&mut source) + .unwrap(); + source.make_root(root); + let advice_key = Word::new([ + Felt::new_unchecked(11), + Felt::new_unchecked(12), + Felt::new_unchecked(13), + Felt::new_unchecked(14), + ]); + let advice_values = vec![Felt::new_unchecked(15), Felt::new_unchecked(16)]; + let source = source.with_advice_map(AdviceMap::from_iter([(advice_key, advice_values)])); + + let mut builder = SparseMastForestBuilder::new(Arc::new(source)); + builder.record_visit(root, VisitKind::FullVisit); + let sparse = builder.finalize(); + let restored = SparseMastForest::read_from_bytes(&sparse.to_bytes()).unwrap(); + + assert!(sparse.advice_map().is_empty()); + assert!(restored.advice_map().is_empty()); +} + +#[test] +fn sparse_reader_rejects_non_empty_advice_map_count() { + let mut bytes = Vec::new(); + 0usize.write_into(&mut bytes); + 0usize.write_into(&mut bytes); + 0usize.write_into(&mut bytes); + 1usize.write_into(&mut bytes); + + let err = SparseMastForest::read_from_bytes(&bytes).unwrap_err(); + + assert!(err.to_string().contains("must not carry advice map entries")); +} + +#[test] +fn sparse_reader_rejects_non_increasing_full_node_ids() { + let block = BasicBlockNodeBuilder::new(vec![Operation::Add]).build().unwrap(); + let mut bytes = Vec::new(); + 0usize.write_into(&mut bytes); + 2usize.write_into(&mut bytes); + write_sparse_block_entry(MastNodeId::from(1), &block, &mut bytes); + write_sparse_block_entry(MastNodeId::from(0), &block, &mut bytes); + 0usize.write_into(&mut bytes); + AdviceMap::default().write_into(&mut bytes); + + let err = SparseMastForest::read_from_bytes(&bytes).unwrap_err(); + assert!(err.to_string().contains("full node ids must be strictly increasing")); +} + +#[test] +fn sparse_reader_rejects_non_increasing_digest_only_ids() { + let block = BasicBlockNodeBuilder::new(vec![Operation::Add]).build().unwrap(); + let mut bytes = Vec::new(); + 0usize.write_into(&mut bytes); + 1usize.write_into(&mut bytes); + write_sparse_block_entry(MastNodeId::from(0), &block, &mut bytes); + 2usize.write_into(&mut bytes); + MastNodeId::from(2).write_into(&mut bytes); + block.digest().write_into(&mut bytes); + MastNodeId::from(1).write_into(&mut bytes); + block.digest().write_into(&mut bytes); + AdviceMap::default().write_into(&mut bytes); + + let err = SparseMastForest::read_from_bytes(&bytes).unwrap_err(); + assert!(err.to_string().contains("digest-only node ids must be strictly increasing")); +} + +#[test] +fn sparse_reader_rejects_trailing_bytes() { + let block = BasicBlockNodeBuilder::new(vec![Operation::Add]).build().unwrap(); + let mut bytes = Vec::new(); + 0usize.write_into(&mut bytes); + 1usize.write_into(&mut bytes); + write_sparse_block_entry(MastNodeId::from(0), &block, &mut bytes); + 0usize.write_into(&mut bytes); + AdviceMap::default().write_into(&mut bytes); + bytes.push(0); + + let err = SparseMastForest::read_from_bytes(&bytes).unwrap_err(); + assert!(err.to_string().contains("extra bytes after SparseMastForest payload")); +} + +#[test] +fn sparse_reader_rejects_dense_payloads() { + let mut forest = MastForest::new(); + let root = BasicBlockNodeBuilder::new(vec![Operation::Add]) + .add_to_forest(&mut forest) + .unwrap(); + forest.make_root(root); + + let result = SparseMastForest::read_from_bytes(&forest.to_bytes()); + assert!(result.is_err()); +} + +#[test] +fn sparse_reader_rejects_oversized_node_id_before_sections() { + let mut bytes = Vec::new(); + 1usize.write_into(&mut bytes); + MastNodeId::from(MastForest::MAX_NODES as u32).write_into(&mut bytes); + + let err = SparseMastForest::read_from_bytes(&bytes).unwrap_err(); + + assert!(err.to_string().contains("procedure root id")); + assert!(err.to_string().contains("number of nodes in the forest")); +} + +#[test] +fn sparse_reader_rejects_oversized_section_count_before_allocation() { + let mut bytes = Vec::new(); + usize::MAX.write_into(&mut bytes); + + let err = SparseMastForest::read_from_bytes(&bytes).unwrap_err(); + + assert!(err.to_string().contains("procedure root count")); +} + +#[test] +fn sparse_serialized_parts_reject_missing_child_digest() { + let (_source, sparse, _true_branch, false_branch, _root) = sparse_split_fixture(); + let nodes = sparse.nodes().iter().map(|(&id, node)| (id, node.clone())).collect(); + let digests = sparse + .digest_entries() + .iter() + .filter_map(|(&id, &digest)| (id != false_branch).then_some((id, digest))) + .collect(); + + let result = SparseMastForest::from_serialized_parts( + nodes, + digests, + sparse.procedure_roots().to_vec(), + sparse.advice_map().clone(), + ); + + assert_matches!( + result, + Err(DeserializationError::InvalidValue(msg)) if msg.contains("without a full node or digest-only entry") + ); +} + +#[test] +fn sparse_serialized_parts_reject_duplicate_full_ids() { + let (_source, sparse, true_branch, _false_branch, _root) = sparse_split_fixture(); + let mut nodes: Vec<_> = sparse.nodes().iter().map(|(&id, node)| (id, node.clone())).collect(); + let duplicate_node = sparse.get_node_by_id(true_branch).unwrap().clone(); + nodes.push((true_branch, duplicate_node)); + let digests = sparse.digest_entries().iter().map(|(&id, &digest)| (id, digest)).collect(); + + let result = SparseMastForest::from_serialized_parts( + nodes, + digests, + sparse.procedure_roots().to_vec(), + sparse.advice_map().clone(), + ); + + assert_matches!( + result, + Err(DeserializationError::InvalidValue(msg)) if msg.contains("duplicate sparse full-node id") + ); +} + +#[test] +fn sparse_serialized_parts_reject_duplicate_digest_only_ids() { + let (_source, sparse, _true_branch, false_branch, _root) = sparse_split_fixture(); + let nodes = sparse.nodes().iter().map(|(&id, node)| (id, node.clone())).collect(); + let mut digests: Vec<_> = + sparse.digest_entries().iter().map(|(&id, &digest)| (id, digest)).collect(); + let digest = sparse.get_digest_by_id(false_branch).unwrap(); + digests.push((false_branch, digest)); + + let result = SparseMastForest::from_serialized_parts( + nodes, + digests, + sparse.procedure_roots().to_vec(), + sparse.advice_map().clone(), + ); + + assert_matches!( + result, + Err(DeserializationError::InvalidValue(msg)) if msg.contains("duplicate sparse digest-only id") + ); +} + +#[test] +fn sparse_serialized_parts_reject_full_digest_overlap() { + let (_source, sparse, true_branch, _false_branch, _root) = sparse_split_fixture(); + let nodes = sparse.nodes().iter().map(|(&id, node)| (id, node.clone())).collect(); + let mut digests: Vec<_> = + sparse.digest_entries().iter().map(|(&id, &digest)| (id, digest)).collect(); + digests.push((true_branch, sparse.get_digest_by_id(true_branch).unwrap())); + + let result = SparseMastForest::from_serialized_parts( + nodes, + digests, + sparse.procedure_roots().to_vec(), + sparse.advice_map().clone(), + ); + + assert_matches!( + result, + Err(DeserializationError::InvalidValue(msg)) if msg.contains("overlaps a digest-only entry") + ); +} + +#[test] +fn sparse_serialized_parts_reject_out_of_range_full_digest_and_root_ids() { + let (_source, sparse, true_branch, false_branch, _root) = sparse_split_fixture(); + let out_of_range = MastNodeId::from(MastForest::MAX_NODES as u32); + + let mut nodes: Vec<_> = sparse.nodes().iter().map(|(&id, node)| (id, node.clone())).collect(); + nodes.push((out_of_range, sparse.get_node_by_id(true_branch).unwrap().clone())); + let digests: Vec<_> = + sparse.digest_entries().iter().map(|(&id, &digest)| (id, digest)).collect(); + let result = SparseMastForest::from_serialized_parts( + nodes, + digests.clone(), + sparse.procedure_roots().to_vec(), + sparse.advice_map().clone(), + ); + assert_matches!( + result, + Err(DeserializationError::InvalidValue(msg)) if msg.contains("full node id") + && msg.contains("exceeds maximum") + ); + + let nodes = sparse.nodes().iter().map(|(&id, node)| (id, node.clone())).collect(); + let mut out_of_range_digests = digests; + out_of_range_digests.push((out_of_range, sparse.get_digest_by_id(false_branch).unwrap())); + let result = SparseMastForest::from_serialized_parts( + nodes, + out_of_range_digests, + sparse.procedure_roots().to_vec(), + sparse.advice_map().clone(), + ); + assert_matches!( + result, + Err(DeserializationError::InvalidValue(msg)) if msg.contains("digest-only node id") + && msg.contains("exceeds maximum") + ); + + let nodes = sparse.nodes().iter().map(|(&id, node)| (id, node.clone())).collect(); + let digests = sparse.digest_entries().iter().map(|(&id, &digest)| (id, digest)).collect(); + let result = SparseMastForest::from_serialized_parts( + nodes, + digests, + vec![out_of_range], + sparse.advice_map().clone(), + ); + assert_matches!( + result, + Err(DeserializationError::InvalidValue(msg)) if msg.contains("procedure root id") + && msg.contains("exceeds maximum") + ); +} + +fn write_sparse_block_entry( + id: MastNodeId, + block: &crate::mast::BasicBlockNode, + target: &mut W, +) { + id.write_into(target); + target.write_u8(0); + block.digest().write_into(target); + + let mut basic_block_data = BasicBlockDataBuilder::new(); + let ops_offset = basic_block_data.encode_basic_block(block); + assert_eq!(ops_offset, 0); + let basic_block_data = basic_block_data.finalize(); + target.write_usize(basic_block_data.len()); + target.write_bytes(&basic_block_data); +} + +fn sparse_payload_ids(bytes: &[u8]) -> (Vec, Vec) { + let mut reader = SliceReader::new(bytes); + + let root_count = reader.read_usize().unwrap(); + for _ in 0..root_count { + let _root = MastNodeId::read_from(&mut reader).unwrap(); + } + + let full_count = reader.read_usize().unwrap(); + let mut full_ids = Vec::new(); + for _ in 0..full_count { + let id = MastNodeId::read_from(&mut reader).unwrap(); + skip_sparse_node(&mut reader); + full_ids.push(id); + } + + let digest_count = reader.read_usize().unwrap(); + let mut digest_ids = Vec::new(); + for _ in 0..digest_count { + let id = MastNodeId::read_from(&mut reader).unwrap(); + let _digest = Word::read_from(&mut reader).unwrap(); + digest_ids.push(id); + } + + (full_ids, digest_ids) +} + +fn skip_sparse_node(reader: &mut SliceReader<'_>) { + let tag = reader.read_u8().unwrap(); + let _digest = Word::read_from(reader).unwrap(); + match tag { + 0 => { + let len = reader.read_usize().unwrap(); + let _ = reader.read_slice(len).unwrap(); + }, + 1 | 2 => { + let _first = MastNodeId::read_from(reader).unwrap(); + let _second = MastNodeId::read_from(reader).unwrap(); + }, + 3..=5 => { + let _child = MastNodeId::read_from(reader).unwrap(); + }, + 6..=8 => {}, + _ => panic!("unexpected sparse test node tag {tag}"), + } +} + /// Test that a forest with a node whose child ids are larger than its own id serializes and /// deserializes successfully. #[test] @@ -1247,7 +1670,7 @@ fn test_deserialize_rejects_unknown_flags() { let bytes = forest.to_bytes(); - for flag in [0x01, 0x04] { + for flag in [0x01, 0x08] { let mut bytes = bytes.clone(); bytes[4] = flag; diff --git a/core/src/mast/sparse.rs b/core/src/mast/sparse.rs index 9ceeafda46..c2897bfd34 100644 --- a/core/src/mast/sparse.rs +++ b/core/src/mast/sparse.rs @@ -1,5 +1,6 @@ use alloc::{ collections::{BTreeMap, BTreeSet}, + string::ToString, sync::Arc, vec::Vec, }; @@ -10,6 +11,8 @@ use crate::{ Word, advice::AdviceMap, mast::{ExecutableMastForest, MastForest, MastNode, MastNodeExt, MastNodeId}, + serde::DeserializationError, + utils::Idx, }; // MAST FOREST ID @@ -50,20 +53,11 @@ pub struct SparseMastForest { /// full-node entry implicitly carries its own digest via [`MastNodeExt::digest`]. digests: BTreeMap, - /// Total number of nodes in the source [`MastForest`] from which this sparse forest was - /// built. Note that this is *not* `nodes.len()` — it is the upper bound on the original - /// [`MastNodeId`] space, preserved so that callers materializing dense-shaped state (e.g. - /// allocating an `IndexVec` keyed by [`MastNodeId`]) know its required size. - num_nodes: usize, - /// Roots of procedures defined within the original MAST forest. roots: Vec, /// Advice map to be loaded into the VM prior to executing procedures from this MAST forest. advice_map: AdviceMap, - - /// Cached commitment to the original MAST forest (i.e. a commitment to all roots). - commitment_cache: Word, } impl SparseMastForest { @@ -73,11 +67,18 @@ impl SparseMastForest { &self.nodes } - /// Returns the total number of nodes in the source [`MastForest`] from which this sparse - /// forest was built. This is *not* the number of visited (i.e. present) nodes — see - /// [`Self::nodes`] for that. + /// Returns the minimum node count needed to cover all IDs retained in this sparse replay view. + /// + /// This is *not* the number of visited nodes and may be smaller than the source + /// [`MastForest`]'s node count when high source IDs were not needed during replay. pub fn num_nodes(&self) -> usize { - self.num_nodes + self.nodes + .keys() + .chain(self.digests.keys()) + .chain(self.roots.iter()) + .map(|id| id.to_usize() + 1) + .max() + .unwrap_or(0) } /// Returns the roots of procedures defined within this sparse forest. @@ -85,19 +86,154 @@ impl SparseMastForest { &self.roots } - /// Returns the advice map associated with this sparse forest. + /// Returns the empty advice map associated with this sparse forest. + /// + /// Sparse replay uses `AdviceReplay` for advice reads; this map remains empty to satisfy the + /// shared [`ExecutableMastForest`] interface. pub fn advice_map(&self) -> &AdviceMap { &self.advice_map } - /// Returns the commitment to this sparse forest, computed from the procedure roots. - /// - /// The commitment value is derived from the digests of the procedure roots in the original - /// forest; it is therefore equal to the commitment of the source [`MastForest`] from which - /// this sparse forest was built. - pub fn commitment(&self) -> Word { - self.commitment_cache + /// Returns the digest-only entries associated with this sparse forest. + pub(in crate::mast) fn digest_entries(&self) -> &BTreeMap { + &self.digests + } + + /// Builds a sparse forest from trusted replay parts. + pub(in crate::mast) fn from_serialized_parts( + nodes: Vec<(MastNodeId, MastNode)>, + digests: Vec<(MastNodeId, Word)>, + roots: Vec, + advice_map: AdviceMap, + ) -> Result { + if !advice_map.is_empty() { + return Err(DeserializationError::InvalidValue( + "sparse MAST replay payload must not carry advice map entries".to_string(), + )); + } + + let nodes = collect_unique_nodes(nodes)?; + let digests = collect_unique_digests(digests)?; + + for &root in &roots { + validate_sparse_id(root, "procedure root")?; + } + + for node_id in nodes.keys() { + if digests.contains_key(node_id) { + return Err(DeserializationError::InvalidValue(format!( + "sparse full-node id {} overlaps a digest-only entry", + node_id.0 + ))); + } + } + + validate_full_node_child_digests(&nodes, &digests)?; + + Ok(Self { + nodes, + digests, + roots, + advice_map: AdviceMap::default(), + }) + } +} + +fn validate_sparse_id(id: MastNodeId, label: &str) -> Result<(), DeserializationError> { + if id.to_usize() >= MastForest::MAX_NODES { + return Err(DeserializationError::InvalidValue(format!( + "{label} id {} exceeds maximum sparse MAST node id {}", + id.0, + MastForest::MAX_NODES - 1 + ))); } + Ok(()) +} + +fn collect_unique_nodes( + nodes: Vec<(MastNodeId, MastNode)>, +) -> Result, DeserializationError> { + let mut result = BTreeMap::new(); + for (id, node) in nodes { + validate_sparse_id(id, "full node")?; + if result.insert(id, node).is_some() { + return Err(DeserializationError::InvalidValue(format!( + "duplicate sparse full-node id {}", + id.0 + ))); + } + } + Ok(result) +} + +fn collect_unique_digests( + digests: Vec<(MastNodeId, Word)>, +) -> Result, DeserializationError> { + let mut result = BTreeMap::new(); + for (id, digest) in digests { + validate_sparse_id(id, "digest-only node")?; + if result.insert(id, digest).is_some() { + return Err(DeserializationError::InvalidValue(format!( + "duplicate sparse digest-only id {}", + id.0 + ))); + } + } + Ok(result) +} + +/// Checks that every child of a retained full node is available as either a full node or a +/// digest-only entry. +fn validate_full_node_child_digests( + nodes: &BTreeMap, + digests: &BTreeMap, +) -> Result<(), DeserializationError> { + for (&node_id, node) in nodes { + validate_sparse_id(node_id, "full node")?; + + match node { + MastNode::Block(block) => { + block.validate_batch_invariants().map_err(|error_msg| { + DeserializationError::InvalidValue(format!( + "invalid sparse basic block {}: {error_msg}", + node_id.0 + )) + })?; + }, + MastNode::External(_) | MastNode::Dyn(_) => {}, + MastNode::Join(join) => { + require_child_digest(node_id, join.first(), nodes, digests)?; + require_child_digest(node_id, join.second(), nodes, digests)?; + }, + MastNode::Split(split) => { + require_child_digest(node_id, split.on_true(), nodes, digests)?; + require_child_digest(node_id, split.on_false(), nodes, digests)?; + }, + MastNode::Loop(loop_node) => { + require_child_digest(node_id, loop_node.body(), nodes, digests)?; + }, + MastNode::Call(call) => { + require_child_digest(node_id, call.callee(), nodes, digests)?; + }, + } + } + Ok(()) +} + +fn require_child_digest( + parent_id: MastNodeId, + child_id: MastNodeId, + nodes: &BTreeMap, + digests: &BTreeMap, +) -> Result<(), DeserializationError> { + validate_sparse_id(child_id, "child")?; + if !nodes.contains_key(&child_id) && !digests.contains_key(&child_id) { + return Err(DeserializationError::InvalidValue(format!( + "sparse full node {} references child {} without a full node or digest-only entry", + parent_id.0, child_id.0 + ))); + } + Ok(()) } impl ExecutableMastForest for SparseMastForest { @@ -162,10 +298,6 @@ pub struct SparseMastForestBuilder { /// The source forest whose nodes are being collected. source: Arc, - /// Total number of nodes in the source forest, captured at construction time. Propagated to - /// the finalized [`SparseMastForest`] so consumers know the original [`MastNodeId`] space. - num_nodes: usize, - /// IDs of nodes that were entered during execution. Their full [`MastNode`] is copied into the /// finalized forest's `nodes` map. full_visits: BTreeSet, @@ -179,10 +311,8 @@ pub struct SparseMastForestBuilder { impl SparseMastForestBuilder { /// Creates a new builder for the given source forest. pub fn new(source: Arc) -> Self { - let num_nodes = source.nodes().len(); Self { source, - num_nodes, full_visits: BTreeSet::new(), digest_only_visits: BTreeSet::new(), } @@ -209,15 +339,10 @@ impl SparseMastForestBuilder { } /// Consumes the builder and produces a [`SparseMastForest`] containing only the visited nodes - /// from the source forest. The roots, advice map, and debug info are cloned from the source - /// in full (they are not yet trimmed to visited nodes only). + /// from the source forest. The roots are cloned from the source in full. Advice data is not + /// copied because sparse replay uses `AdviceReplay`. pub fn finalize(self) -> SparseMastForest { - let SparseMastForestBuilder { - source, - num_nodes, - full_visits, - digest_only_visits, - } = self; + let SparseMastForestBuilder { source, full_visits, digest_only_visits } = self; let mut nodes = BTreeMap::new(); for node_id in &full_visits { @@ -241,10 +366,8 @@ impl SparseMastForestBuilder { SparseMastForest { nodes, digests, - num_nodes, roots: source.procedure_roots().to_vec(), - advice_map: source.advice_map().clone(), - commitment_cache: source.commitment(), + advice_map: AdviceMap::default(), } } }