From 49829a8050c177e32dc96f3c64c45350907b95fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Garillot?= Date: Tue, 30 Jun 2026 19:05:41 -0400 Subject: [PATCH 1/9] Serialize sparse MAST forests --- core/src/mast/mod.rs | 22 +- core/src/mast/serialization/layout.rs | 15 +- core/src/mast/serialization/mod.rs | 26 +- core/src/mast/serialization/sparse.rs | 402 +++++++++++++++++++++ core/src/mast/serialization/tests.rs | 497 +++++++++++++++++++++++++- core/src/mast/sparse.rs | 157 ++++++++ 6 files changed, 1102 insertions(+), 17 deletions(-) create mode 100644 core/src/mast/serialization/sparse.rs diff --git a/core/src/mast/mod.rs b/core/src/mast/mod.rs index 582f572776..134e4ec3e2 100644 --- a/core/src/mast/mod.rs +++ b/core/src/mast/mod.rs @@ -54,7 +54,7 @@ use proptest::prelude::*; use serde::{Deserialize, Serialize}; #[cfg(feature = "serde")] -use crate::serde::{Deserializable, SliceReader}; +use crate::serde::SliceReader; mod node; #[cfg(any(test, feature = "arbitrary"))] @@ -71,14 +71,14 @@ use crate::{ Felt, Word, advice::AdviceMap, crypto::hash::Poseidon2, - serde::{ByteWriter, DeserializationError, Serializable}, + serde::{ByteWriter, Deserializable, DeserializationError, Serializable}, utils::{DenseIdMap, Idx, IndexVec, hash_string_to_word}, }; mod serialization; pub use serialization::{ AdviceMapView, AdviceValueView, MastForestReadMode, MastForestReadView, MastForestView, - MastForestWireView, MastNodeEntry, MastNodeInfo, + MastForestWireView, MastNodeEntry, MastNodeInfo, SparseMastForestReadOptions, }; mod dense_builder; @@ -1168,6 +1168,10 @@ impl ExecutableMastForest for Arc { #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(transparent))] +#[cfg_attr( + all(feature = "arbitrary", test), + miden_test_serde_macros::serde_test(binary_serde(true)) +)] pub struct MastNodeId(u32); /// Operations that mutate a MAST often produce this mapping between old and new NodeIds. @@ -1242,6 +1246,18 @@ impl Serializable for MastNodeId { } } +impl Deserializable for MastNodeId { + fn read_from( + source: &mut R, + ) -> Result { + Ok(Self(::read_from(source)?)) + } + + fn min_serialized_size() -> usize { + ::min_serialized_size() + } +} + impl fmt::Display for MastNodeId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "MastNodeId({})", self.0) diff --git a/core/src/mast/serialization/layout.rs b/core/src/mast/serialization/layout.rs index 95fab0d2ca..b790e593b1 100644 --- a/core/src/mast/serialization/layout.rs +++ b/core/src/mast/serialization/layout.rs @@ -1,6 +1,8 @@ use alloc::{format, string::ToString, vec::Vec}; -use super::{FLAG_HASHLESS, FLAGS_RESERVED_MASK, MAGIC, MastForest, MastNodeEntry, VERSION}; +use super::{ + FLAG_HASHLESS, FLAG_SPARSE, FLAGS_RESERVED_MASK, MAGIC, MastForest, MastNodeEntry, VERSION, +}; use crate::{ mast::MastNodeId, serde::{ByteReader, Deserializable, DeserializationError, SliceReader}, @@ -157,6 +159,10 @@ impl WireFlags { pub(super) fn is_hashless(self) -> bool { self.0 & FLAG_HASHLESS != 0 } + + pub(super) fn is_sparse(self) -> bool { + self.0 & FLAG_SPARSE != 0 + } } // LAYOUT SCANNING @@ -171,6 +177,11 @@ pub(super) fn read_header_and_scan_layout( // untrusted deserialization path. let (raw_flags, _version) = read_and_validate_header(source)?; let flags = WireFlags::new(raw_flags); + if flags.is_sparse() { + return Err(DeserializationError::InvalidValue( + "SPARSE flag is set; use SparseMastForest for sparse replay input".to_string(), + )); + } if flags.is_hashless() && !allow_hashless { return Err(DeserializationError::InvalidValue( "HASHLESS flag is set; use UntrustedMastForest for untrusted input".to_string(), @@ -457,7 +468,7 @@ fn validate_budgeted_count( Ok(()) } -fn read_and_validate_header( +pub(super) fn read_and_validate_header( source: &mut R, ) -> Result<(u8, [u8; 3]), DeserializationError> { let magic: [u8; 4] = source.read_array()?; diff --git a/core/src/mast/serialization/mod.rs b/core/src/mast/serialization/mod.rs index a50d765d2b..e70690164d 100644 --- a/core/src/mast/serialization/mod.rs +++ b/core/src/mast/serialization/mod.rs @@ -76,9 +76,14 @@ //! same contiguous array on the wire. //! //! Public entry points adopt these policies: -//! - [`MastForest::read_from_bytes`]: trusted execution payload, no hashless support. +//! - [`MastForest::read_from_bytes`]: trusted dense execution payload, no hashless or sparse +//! support. //! - [`MastForestWireView::new`]: trusted wire-backed cache access; rejects hashless and legacy -//! debug-bearing payloads. +//! debug-bearing payloads, and rejects sparse payloads. +//! - [`crate::mast::SparseMastForest::read_from_bytes`] / +//! [`crate::mast::SparseMastForest::read_from_bytes_with_options`]: trusted sparse replay +//! payloads for serialized trace-generation inputs. Sparse payloads currently carry full-node +//! digests and do not recompute them on read. //! - [`crate::mast::UntrustedMastForest::read_from_bytes`] / //! [`crate::mast::UntrustedMastForest::read_from_bytes_with_options`]: untrusted parsing plus //! later validation before use. @@ -112,6 +117,9 @@ mod layout; pub(super) use layout::ForestLayout; use layout::{OffsetTrackingReader, TrackingReader, WireFlags, read_header_and_scan_layout}; +mod sparse; +pub use sparse::SparseMastForestReadOptions; + mod resolved; use resolved::{ResolvedSerializedForest, basic_block_offset_for_node_index}; @@ -171,10 +179,16 @@ const MAGIC: &[u8; 4] = b"MAST"; /// from local structure. pub(super) const FLAG_HASHLESS: u8 = 0x02; +/// Flag indicating that the payload uses sparse MAST replay serialization. +/// +/// Sparse payloads preserve the source forest's [`MastNodeId`] space and therefore cannot be read +/// through dense [`MastForest`] entry points. +pub(super) const FLAG_SPARSE: u8 = 0x04; + /// Mask for reserved flag bits that must be zero. /// -/// Bit 0 and bits 2-7 are reserved for future use. If any are set, deserialization fails. -const FLAGS_RESERVED_MASK: u8 = 0xfd; +/// Bit 0 and bits 3-7 are reserved for future use. If any are set, deserialization fails. +const FLAGS_RESERVED_MASK: u8 = 0xf9; /// The format version. /// @@ -201,13 +215,15 @@ const FLAGS_RESERVED_MASK: u8 = 0xfd; /// records. MAST nodes are metadata-free identifiers. Before any public release on this branch, /// the same unreleased wire version also reserved bit 0 and stopped using it as a forest-level /// debug-presence flag. +/// - [0, 0, 5]: Added SPARSE flag (bit 2). Sparse payloads preserve sparse replay IDs and are +/// accepted only by SparseMastForest readers. /// /// Legacy wire versions (pre-#3192 decorator terminology): /// [0,0,1] stored metadata as serialized decorator variants in CSR per-node slots. /// [0,0,2] removed AssemblyOp from the decorator enum and stored them separately in DebugInfo. /// [0,0,3] removed the unused decorator-count wire field. /// [0,0,4] eliminated the decorator wire slots entirely. -const VERSION: [u8; 3] = [0, 0, 4]; +const VERSION: [u8; 3] = [0, 0, 5]; // MAST FOREST SERIALIZATION/DESERIALIZATION // ================================================================================================ diff --git a/core/src/mast/serialization/sparse.rs b/core/src/mast/serialization/sparse.rs new file mode 100644 index 0000000000..63d032061a --- /dev/null +++ b/core/src/mast/serialization/sparse.rs @@ -0,0 +1,402 @@ +use alloc::{format, string::ToString, vec::Vec}; + +use super::{ + FLAG_HASHLESS, FLAG_SPARSE, MAGIC, MastNodeEntry, VERSION, + basic_blocks::{BasicBlockDataBuilder, BasicBlockDataDecoder}, + layout::{OffsetTrackingReader, TrackingReader, WireFlags, read_and_validate_header}, +}; +use crate::{ + Word, + advice::AdviceMap, + mast::{MastForest, MastNode, MastNodeExt, MastNodeId, SparseMastForest}, + serde::{ + BudgetedReader, ByteReader, ByteWriter, Deserializable, DeserializationError, Serializable, + SliceReader, + }, +}; + +const SPARSE_FLAGS: u8 = FLAG_HASHLESS | FLAG_SPARSE; + +fn sparse_mast_forest_min_serialized_size() -> usize { + MAGIC.len() + + 1 + + VERSION.len() + + usize::min_serialized_size() * 7 + + Word::min_serialized_size() + + usize::min_serialized_size() +} + +/// Serializes a [`SparseMastForest`] in trusted sparse replay form. +/// +/// This format carries the digest for each full node and accepts those digests on read. It is +/// suitable for trusted remote proving inputs, not as an untrusted hashless validation path. +/// +/// See for the planned untrusted reader. +pub(super) fn write_sparse_into(forest: &SparseMastForest, target: &mut W) { + let mut basic_block_data_builder = BasicBlockDataBuilder::new(); + let mut full_ids = Vec::with_capacity(forest.nodes().len()); + let mut entries = Vec::with_capacity(forest.nodes().len()); + let mut full_digests = Vec::with_capacity(forest.nodes().len()); + + for (&node_id, node) in forest.nodes() { + let ops_offset = if let MastNode::Block(basic_block) = node { + basic_block_data_builder.encode_basic_block(basic_block) + } else { + 0 + }; + + full_ids.push(node_id); + entries.push(MastNodeEntry::new(node, ops_offset)); + full_digests.push(node.digest()); + } + + let basic_block_data = basic_block_data_builder.finalize(); + let external_full_node_count = + entries.iter().filter(|entry| matches!(entry, MastNodeEntry::External)).count(); + let non_external_count = + entries.iter().filter(|entry| !matches!(entry, MastNodeEntry::External)).count(); + + target.write_bytes(MAGIC); + target.write_u8(SPARSE_FLAGS); + target.write_bytes(&VERSION); + + target.write_usize(forest.procedure_roots().len()); + target.write_usize(forest.num_nodes()); + target.write_usize(full_ids.len()); + target.write_usize(forest.digest_entries().len()); + target.write_usize(external_full_node_count); + target.write_usize(non_external_count); + target.write_usize(basic_block_data.len()); + + for &root in forest.procedure_roots() { + root.0.write_into(target); + } + + forest.commitment().write_into(target); + target.write_bytes(&basic_block_data); + + for id in full_ids { + id.0.write_into(target); + } + + for entry in entries { + entry.write_into(target); + } + + for digest in full_digests { + digest.write_into(target); + } + + for (&id, &digest) in forest.digest_entries() { + id.0.write_into(target); + digest.write_into(target); + } + + forest.advice_map().write_into(target); +} + +impl Serializable for SparseMastForest { + fn write_into(&self, target: &mut W) { + write_sparse_into(self, target); + } +} + +impl Deserializable for SparseMastForest { + /// Reads a trusted sparse replay payload. + /// + /// Full-node digests are accepted from the payload. This is not the untrusted hash-validation + /// path from . + fn read_from(source: &mut R) -> Result { + read_sparse_from(source) + } + + fn min_serialized_size() -> usize { + sparse_mast_forest_min_serialized_size() + } + + /// Reads trusted sparse replay bytes and rejects trailing bytes. + fn read_from_bytes(bytes: &[u8]) -> Result { + SparseMastForest::read_from_bytes(bytes) + } +} + +/// Reads a trusted sparse replay payload. +/// +/// The payload carries full-node digests and digest-only entries as replay data. It does not +/// rebuild those hashes from node structure. +pub(super) fn read_sparse_from( + source: &mut R, +) -> Result { + let mut reader = TrackingReader::new(source); + let (raw_flags, _version) = read_and_validate_header(&mut reader)?; + let flags = WireFlags::new(raw_flags); + validate_sparse_flags(flags)?; + + let root_count = read_bounded_count(&mut reader, size_of::(), "procedure root count")?; + let source_node_count = reader.read_usize()?; + if source_node_count > MastForest::MAX_NODES { + return Err(DeserializationError::InvalidValue(format!( + "source node count {source_node_count} exceeds maximum allowed {}", + MastForest::MAX_NODES + ))); + } + + let full_node_count = read_bounded_count( + &mut reader, + MastNodeEntry::SERIALIZED_SIZE + Word::min_serialized_size(), + "full node count", + )?; + let digest_only_count = read_bounded_count( + &mut reader, + size_of::() + Word::min_serialized_size(), + "digest-only node count", + )?; + let external_full_node_count = read_bounded_count( + &mut reader, + MastNodeEntry::SERIALIZED_SIZE, + "external full-node count", + )?; + let non_external_full_node_count = read_bounded_count( + &mut reader, + MastNodeEntry::SERIALIZED_SIZE, + "non-external full-node count", + )?; + let basic_block_data_len = read_bounded_count(&mut reader, 1, "basic-block data length")?; + + let counted_full = external_full_node_count + .checked_add(non_external_full_node_count) + .ok_or_else(|| { + DeserializationError::InvalidValue("full node count overflow".to_string()) + })?; + if counted_full != full_node_count { + return Err(DeserializationError::InvalidValue(format!( + "sparse header full node count {full_node_count} does not match external + non-external count {counted_full}" + ))); + } + + let roots = read_id_section(&mut reader, root_count, source_node_count, "procedure root")?; + let commitment = Word::read_from(&mut reader)?; + let basic_block_data = reader.read_slice(basic_block_data_len)?.to_vec(); + let full_ids = read_id_section(&mut reader, full_node_count, source_node_count, "full node")?; + validate_strictly_increasing_ids(&full_ids, "full node")?; + + let mut entries = Vec::with_capacity(full_node_count); + for _ in 0..full_node_count { + entries.push(MastNodeEntry::read_from(&mut reader)?); + } + + let counted_external = + entries.iter().filter(|entry| matches!(entry, MastNodeEntry::External)).count(); + if counted_external != external_full_node_count { + return Err(DeserializationError::InvalidValue(format!( + "sparse header external full-node count {external_full_node_count} does not match {counted_external} external entries" + ))); + } + + let mut full_digests = Vec::with_capacity(full_node_count); + for _ in 0..full_node_count { + full_digests.push(Word::read_from(&mut reader)?); + } + + let mut digest_entries = Vec::with_capacity(digest_only_count); + for _ in 0..digest_only_count { + let id = read_node_id(&mut reader, source_node_count, "digest-only node")?; + let digest = Word::read_from(&mut reader)?; + digest_entries.push((id, digest)); + } + validate_strictly_increasing_entry_ids(&digest_entries, "digest-only node")?; + + let advice_map = AdviceMap::read_from(&mut reader)?; + let nodes = materialize_sparse_nodes( + &full_ids, + &entries, + &full_digests, + source_node_count, + &basic_block_data, + )?; + + SparseMastForest::from_serialized_parts( + nodes, + digest_entries, + source_node_count, + roots, + advice_map, + commitment, + ) +} + +fn validate_sparse_flags(flags: WireFlags) -> Result<(), DeserializationError> { + if !flags.is_sparse() { + return Err(DeserializationError::InvalidValue( + "SPARSE flag is not set; use MastForest readers for dense input".to_string(), + )); + } + if !flags.is_hashless() { + return Err(DeserializationError::InvalidValue( + "sparse MAST payloads must also set HASHLESS".to_string(), + )); + } + if flags.bits() != SPARSE_FLAGS { + return Err(DeserializationError::InvalidValue(format!( + "invalid sparse MAST flag combination: {:#04x}", + flags.bits() + ))); + } + Ok(()) +} + +fn read_bounded_count( + source: &mut R, + element_size: usize, + label: &str, +) -> Result { + let count = source.read_usize()?; + let max_count = source.max_alloc(element_size); + if count > max_count { + return Err(DeserializationError::InvalidValue(format!( + "{label} {count} exceeds reader allocation bound {max_count} for {element_size}-byte elements" + ))); + } + Ok(count) +} + +fn read_id_section( + source: &mut R, + count: usize, + node_count: usize, + label: &str, +) -> Result, DeserializationError> { + let mut ids = Vec::with_capacity(count); + for _ in 0..count { + ids.push(read_node_id(source, node_count, label)?); + } + Ok(ids) +} + +fn read_node_id( + source: &mut R, + node_count: usize, + label: &str, +) -> Result { + let raw = u32::read_from(source)?; + MastNodeId::from_u32_with_node_count(raw, node_count).map_err(|err| { + DeserializationError::InvalidValue(format!("invalid {label} id {raw}: {err}")) + }) +} + +fn validate_strictly_increasing_ids( + ids: &[MastNodeId], + label: &str, +) -> Result<(), DeserializationError> { + for pair in ids.windows(2) { + if pair[0].0 >= pair[1].0 { + return Err(DeserializationError::InvalidValue(format!( + "{label} ids must be strictly increasing" + ))); + } + } + Ok(()) +} + +fn validate_strictly_increasing_entry_ids( + entries: &[(MastNodeId, Word)], + label: &str, +) -> Result<(), DeserializationError> { + for pair in entries.windows(2) { + if pair[0].0.0 >= pair[1].0.0 { + return Err(DeserializationError::InvalidValue(format!( + "{label} ids must be strictly increasing" + ))); + } + } + Ok(()) +} + +fn materialize_sparse_nodes( + full_ids: &[MastNodeId], + entries: &[MastNodeEntry], + full_digests: &[Word], + source_node_count: usize, + basic_block_data: &[u8], +) -> Result, DeserializationError> { + let basic_block_data_decoder = BasicBlockDataDecoder::new(basic_block_data); + if full_digests.len() != full_ids.len() { + return Err(DeserializationError::InvalidValue(format!( + "sparse full digest count {} does not match full node count {}", + full_digests.len(), + full_ids.len() + ))); + } + + let mut nodes = Vec::with_capacity(entries.len()); + for ((&node_id, &entry), &digest) in full_ids.iter().zip(entries).zip(full_digests) { + let node = entry + .try_into_mast_node_builder(source_node_count, &basic_block_data_decoder, digest)? + .build_linked() + .map_err(|err| { + DeserializationError::InvalidValue(format!( + "failed to build sparse MAST node {}: {err}", + node_id.0 + )) + })?; + nodes.push((node_id, node)); + } + + Ok(nodes) +} + +impl SparseMastForest { + /// Deserializes trusted sparse MAST replay bytes using default parse budgets. + /// + /// This reader bounds parsing, but accepts sparse MAST hashes from the payload. + pub fn read_from_bytes(bytes: &[u8]) -> Result { + Self::read_from_bytes_with_options(bytes, SparseMastForestReadOptions::default()) + } + + /// Deserializes trusted sparse MAST replay bytes using explicit read options. + /// + /// See for the planned untrusted reader. + pub fn read_from_bytes_with_options( + bytes: &[u8], + options: SparseMastForestReadOptions, + ) -> Result { + let wire_byte_budget = options.wire_byte_budget(bytes.len()); + if wire_byte_budget < bytes.len() { + return Err(DeserializationError::InvalidValue( + "SparseMastForest wire byte budget is smaller than payload length".to_string(), + )); + } + let allocation_budget = wire_byte_budget.min(bytes.len().saturating_mul(4)); + let mut reader = BudgetedReader::new(SliceReader::new(bytes), allocation_budget); + let forest = read_sparse_from(&mut reader)?; + if reader.has_more_bytes() { + return Err(DeserializationError::InvalidValue( + "extra bytes after SparseMastForest payload".to_string(), + )); + } + Ok(forest) + } +} + +/// Options for reading a [`SparseMastForest`] from bytes. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct SparseMastForestReadOptions { + wire_byte_budget: Option, +} + +impl SparseMastForestReadOptions { + /// Creates options that use the default sparse read budgets. + pub fn new() -> Self { + Self::default() + } + + /// Sets the maximum number of serialized bytes consumed while parsing wire data. + pub fn with_wire_byte_budget(mut self, budget: usize) -> Self { + self.wire_byte_budget = Some(budget); + self + } + + fn wire_byte_budget(self, bytes_len: usize) -> usize { + self.wire_byte_budget.unwrap_or(bytes_len) + } +} diff --git a/core/src/mast/serialization/tests.rs b/core/src/mast/serialization/tests.rs index 39dd5de02d..2e2eb88266 100644 --- a/core/src/mast/serialization/tests.rs +++ b/core/src/mast/serialization/tests.rs @@ -1,7 +1,7 @@ use core::assert_matches; use std::{ string::{String, ToString}, - sync::{Mutex, Once}, + sync::{Arc, Mutex, Once}, }; use super::*; @@ -9,10 +9,11 @@ use crate::{ Felt, Word, chiplets::hasher, mast::{ - BasicBlockNodeBuilder, CallNodeBuilder, DynNodeBuilder, ExternalNodeBuilder, - JoinNodeBuilder, LoopNodeBuilder, MastForestError, MastForestView, MastNodeExt, MastNodeId, - OP_BATCH_SIZE, OpBatch, SplitNodeBuilder, UntrustedMastForest, - UntrustedMastForestReadOptions, + BasicBlockNodeBuilder, CallNodeBuilder, DynNodeBuilder, ExecutableMastForest, + ExternalNodeBuilder, JoinNodeBuilder, LoopNodeBuilder, MastForestContributor, + MastForestError, MastForestView, MastNodeExt, MastNodeId, OP_BATCH_SIZE, OpBatch, + SparseMastForest, SparseMastForestBuilder, SparseMastForestReadOptions, SplitNodeBuilder, + UntrustedMastForest, UntrustedMastForestReadOptions, VisitKind, }, operations::Operation, serde::{ByteReader, Deserializable, DeserializationError, Serializable, SliceReader}, @@ -746,6 +747,488 @@ fn test_untrusted_hashless_keeps_external_digests_by_prefix() { assert_eq!(restored[MastNodeId::new_unchecked(1)].digest(), external_high); } +fn sparse_split_fixture() -> (Arc, SparseMastForest, MastNodeId, MastNodeId, MastNodeId) +{ + let mut forest = MastForest::new(); + let true_branch = BasicBlockNodeBuilder::new(vec![Operation::Add]) + .add_to_forest(&mut forest) + .unwrap(); + let false_branch = BasicBlockNodeBuilder::new(vec![Operation::Mul]) + .add_to_forest(&mut forest) + .unwrap(); + let root = SplitNodeBuilder::new([true_branch, false_branch]) + .add_to_forest(&mut forest) + .unwrap(); + forest.make_root(root); + + let forest = Arc::new(forest); + let mut builder = SparseMastForestBuilder::new(Arc::clone(&forest)); + builder.record_visit(true_branch, VisitKind::FullVisit); + builder.record_visit(false_branch, VisitKind::DigestOnly); + builder.record_visit(root, VisitKind::FullVisit); + let sparse = builder.finalize(); + + (forest, sparse, true_branch, false_branch, root) +} + +#[test] +fn sparse_mast_round_trip_preserves_sparse_replay_ids() { + let (source, sparse, true_branch, false_branch, root) = sparse_split_fixture(); + + let bytes = sparse.to_bytes(); + let restored = SparseMastForest::read_from_bytes(&bytes).unwrap(); + + assert_eq!(restored.num_nodes(), source.num_nodes() as usize); + assert_eq!(restored.procedure_roots(), &[root]); + assert_eq!(restored.commitment(), source.commitment()); + assert_eq!( + restored.get_node_by_id(true_branch).unwrap().digest(), + source[true_branch].digest() + ); + assert_eq!(restored.get_node_by_id(root).unwrap().digest(), source[root].digest()); + assert!(restored.get_node_by_id(false_branch).is_none()); + assert_eq!(restored.get_digest_by_id(true_branch), Some(source[true_branch].digest())); + assert_eq!(restored.get_digest_by_id(false_branch), Some(source[false_branch].digest())); + assert_eq!(restored.get_digest_by_id(root), Some(source[root].digest())); +} + +#[test] +fn sparse_mast_round_trip_preserves_external_full_node() { + let mut forest = MastForest::new(); + let unvisited = BasicBlockNodeBuilder::new(vec![Operation::Add]) + .add_to_forest(&mut forest) + .unwrap(); + let external_digest = Word::new([ + Felt::new_unchecked(17), + Felt::new_unchecked(18), + Felt::new_unchecked(19), + Felt::new_unchecked(20), + ]); + let external = ExternalNodeBuilder::new(external_digest).add_to_forest(&mut forest).unwrap(); + forest.make_root(external); + + let forest = Arc::new(forest); + let mut builder = SparseMastForestBuilder::new(Arc::clone(&forest)); + builder.record_visit(external, VisitKind::FullVisit); + let sparse = builder.finalize(); + + let restored = SparseMastForest::read_from_bytes(&sparse.to_bytes()).unwrap(); + + assert_eq!(restored.num_nodes(), forest.num_nodes() as usize); + assert_eq!(restored.procedure_roots(), &[external]); + assert_eq!(restored.commitment(), forest.commitment()); + assert_eq!(restored.get_node_by_id(external).unwrap().digest(), external_digest); + assert_eq!(restored.get_digest_by_id(external), Some(external_digest)); + assert!(restored.get_node_by_id(unvisited).is_none()); + assert_eq!(restored.get_digest_by_id(unvisited), None); +} + +fn write_sparse_test_payload( + source_node_count: usize, + roots: &[MastNodeId], + full_ids: &[MastNodeId], + entries: &[MastNodeEntry], + full_digests: &[Word], + basic_block_data: &[u8], + commitment: Word, +) -> Vec { + assert_eq!(full_ids.len(), entries.len()); + assert_eq!(full_ids.len(), full_digests.len()); + + let mut bytes = Vec::new(); + bytes.write_bytes(MAGIC); + bytes.write_u8(FLAG_HASHLESS | FLAG_SPARSE); + bytes.write_bytes(&VERSION); + + bytes.write_usize(roots.len()); + bytes.write_usize(source_node_count); + bytes.write_usize(full_ids.len()); + bytes.write_usize(0); + bytes.write_usize( + entries.iter().filter(|entry| matches!(entry, MastNodeEntry::External)).count(), + ); + bytes.write_usize(full_ids.len()); + bytes.write_usize(basic_block_data.len()); + + for root in roots { + root.0.write_into(&mut bytes); + } + commitment.write_into(&mut bytes); + bytes.write_bytes(basic_block_data); + for id in full_ids { + id.0.write_into(&mut bytes); + } + for entry in entries { + entry.write_into(&mut bytes); + } + for digest in full_digests { + digest.write_into(&mut bytes); + } + AdviceMap::default().write_into(&mut bytes); + bytes +} + +#[test] +fn sparse_reader_allows_large_source_node_count_with_small_payload() { + let block = BasicBlockNodeBuilder::new(vec![Operation::Add]).build().unwrap(); + let mut basic_block_data = BasicBlockDataBuilder::new(); + let block_offset = basic_block_data.encode_basic_block(&block); + let basic_block_data = basic_block_data.finalize(); + + let root = MastNodeId::from(0); + let bytes = write_sparse_test_payload( + MastForest::MAX_NODES, + &[root], + &[root], + &[MastNodeEntry::Block { ops_offset: block_offset }], + &[block.digest()], + &basic_block_data, + block.digest(), + ); + + let restored = SparseMastForest::read_from_bytes(&bytes).unwrap(); + assert_eq!(restored.num_nodes(), MastForest::MAX_NODES); + assert_eq!(restored.get_digest_by_id(root), Some(block.digest())); +} + +#[test] +fn sparse_reader_reconstructs_forward_full_child_digests() { + let left_block = BasicBlockNodeBuilder::new(vec![Operation::Add]).build().unwrap(); + let right_block = BasicBlockNodeBuilder::new(vec![Operation::Mul]).build().unwrap(); + + let mut basic_block_data = BasicBlockDataBuilder::new(); + let left_offset = basic_block_data.encode_basic_block(&left_block); + let right_offset = basic_block_data.encode_basic_block(&right_block); + let basic_block_data = basic_block_data.finalize(); + + let root = MastNodeId::from(0); + let left = MastNodeId::from(1); + let right = MastNodeId::from(2); + let expected_root_digest = hasher::merge_in_domain( + &[left_block.digest(), right_block.digest()], + crate::mast::JoinNode::DOMAIN, + ); + let bytes = write_sparse_test_payload( + 3, + &[root], + &[root, left, right], + &[ + MastNodeEntry::Join { + left_child_id: left.0, + right_child_id: right.0, + }, + MastNodeEntry::Block { ops_offset: left_offset }, + MastNodeEntry::Block { ops_offset: right_offset }, + ], + &[expected_root_digest, left_block.digest(), right_block.digest()], + &basic_block_data, + expected_root_digest, + ); + + let restored = SparseMastForest::read_from_bytes(&bytes).unwrap(); + assert_eq!(restored.get_digest_by_id(root), Some(expected_root_digest)); + assert_eq!(restored.get_digest_by_id(left), Some(left_block.digest())); + assert_eq!(restored.get_digest_by_id(right), Some(right_block.digest())); +} + +#[test] +fn sparse_reader_preserves_forced_full_node_digest() { + let child_block = BasicBlockNodeBuilder::new(vec![Operation::Add]).build().unwrap(); + let mut basic_block_data = BasicBlockDataBuilder::new(); + let child_offset = basic_block_data.encode_basic_block(&child_block); + let basic_block_data = basic_block_data.finalize(); + + let root = MastNodeId::from(0); + let child = MastNodeId::from(1); + let canonical_root_digest = hasher::merge_in_domain( + &[child_block.digest(), Word::default()], + crate::mast::CallNode::CALL_DOMAIN, + ); + let forced_root_digest = Word::new([ + Felt::from(101_u32), + Felt::from(102_u32), + Felt::from(103_u32), + Felt::from(104_u32), + ]); + assert_ne!(forced_root_digest, canonical_root_digest); + + let bytes = write_sparse_test_payload( + 2, + &[root], + &[root, child], + &[ + MastNodeEntry::Call { callee_id: child.0 }, + MastNodeEntry::Block { ops_offset: child_offset }, + ], + &[forced_root_digest, child_block.digest()], + &basic_block_data, + forced_root_digest, + ); + + let restored = SparseMastForest::read_from_bytes(&bytes).unwrap(); + assert_eq!(restored.get_digest_by_id(root), Some(forced_root_digest)); + assert_eq!(restored.commitment(), forced_root_digest); +} + +#[test] +fn sparse_reader_reconstructs_deep_forward_full_child_chain() { + const CHAIN_LEN: usize = 4096; + + let block = BasicBlockNodeBuilder::new(vec![Operation::Add]).build().unwrap(); + let mut basic_block_data = BasicBlockDataBuilder::new(); + let block_offset = basic_block_data.encode_basic_block(&block); + let basic_block_data = basic_block_data.finalize(); + + let full_ids: Vec<_> = (0..CHAIN_LEN).map(|id| MastNodeId::from(id as u32)).collect(); + let mut entries = Vec::with_capacity(CHAIN_LEN); + for id in 0..CHAIN_LEN - 1 { + entries.push(MastNodeEntry::Call { callee_id: (id + 1) as u32 }); + } + entries.push(MastNodeEntry::Block { ops_offset: block_offset }); + + let mut full_digests = vec![Word::default(); CHAIN_LEN]; + full_digests[CHAIN_LEN - 1] = block.digest(); + for id in (0..CHAIN_LEN - 1).rev() { + full_digests[id] = hasher::merge_in_domain( + &[full_digests[id + 1], Word::default()], + crate::mast::CallNode::CALL_DOMAIN, + ); + } + let expected_root_digest = full_digests[0]; + + let root = MastNodeId::from(0); + let bytes = write_sparse_test_payload( + CHAIN_LEN, + &[root], + &full_ids, + &entries, + &full_digests, + &basic_block_data, + expected_root_digest, + ); + + let restored = SparseMastForest::read_from_bytes(&bytes).unwrap(); + assert_eq!(restored.get_digest_by_id(root), Some(expected_root_digest)); +} + +#[test] +fn sparse_reader_rejects_trailing_bytes_with_exact_prefix_budget() { + let block = BasicBlockNodeBuilder::new(vec![Operation::Add]).build().unwrap(); + let mut basic_block_data = BasicBlockDataBuilder::new(); + let block_offset = basic_block_data.encode_basic_block(&block); + let basic_block_data = basic_block_data.finalize(); + + let root = MastNodeId::from(0); + let mut bytes_with_trailing = write_sparse_test_payload( + 1, + &[root], + &[root], + &[MastNodeEntry::Block { ops_offset: block_offset }], + &[block.digest()], + &basic_block_data, + block.digest(), + ); + + bytes_with_trailing.push(0); + let err = SparseMastForest::read_from_bytes_with_options( + &bytes_with_trailing, + SparseMastForestReadOptions::new().with_wire_byte_budget(bytes_with_trailing.len()), + ) + .unwrap_err(); + assert!(err.to_string().contains("extra bytes after SparseMastForest payload")); +} + +#[test] +fn dense_mast_readers_reject_sparse_payloads() { + let (_source, sparse, _true_branch, _false_branch, _root) = sparse_split_fixture(); + let bytes = sparse.to_bytes(); + + let materialized = MastForest::read_from_bytes(&bytes); + assert_matches!( + materialized, + Err(DeserializationError::InvalidValue(msg)) if msg.contains("SPARSE flag is set") + ); + + let wire_view = MastForestWireView::new(&bytes); + assert_matches!( + wire_view, + Err(DeserializationError::InvalidValue(msg)) if msg.contains("SPARSE flag is set") + ); + + let untrusted = UntrustedMastForest::read_from_bytes(&bytes); + assert_matches!( + untrusted, + Err(DeserializationError::InvalidValue(msg)) if msg.contains("SPARSE flag is set") + ); +} + +#[test] +fn sparse_reader_rejects_dense_payloads() { + let mut forest = MastForest::new(); + let root = BasicBlockNodeBuilder::new(vec![Operation::Add]) + .add_to_forest(&mut forest) + .unwrap(); + forest.make_root(root); + + let result = SparseMastForest::read_from_bytes(&forest.to_bytes()); + assert_matches!( + result, + Err(DeserializationError::InvalidValue(msg)) if msg.contains("SPARSE flag is not set") + ); +} + +#[test] +fn sparse_serialized_parts_reject_missing_child_digest() { + let (_source, sparse, _true_branch, false_branch, _root) = sparse_split_fixture(); + let nodes = sparse.nodes().iter().map(|(&id, node)| (id, node.clone())).collect(); + let digests = sparse + .digest_entries() + .iter() + .filter_map(|(&id, &digest)| (id != false_branch).then_some((id, digest))) + .collect(); + + let result = SparseMastForest::from_serialized_parts( + nodes, + digests, + sparse.num_nodes(), + sparse.procedure_roots().to_vec(), + sparse.advice_map().clone(), + sparse.commitment(), + ); + + assert_matches!( + result, + Err(DeserializationError::InvalidValue(msg)) if msg.contains("without a full node or digest-only entry") + ); +} + +#[test] +fn sparse_serialized_parts_reject_duplicate_full_ids() { + let (_source, sparse, true_branch, _false_branch, _root) = sparse_split_fixture(); + let mut nodes: Vec<_> = sparse.nodes().iter().map(|(&id, node)| (id, node.clone())).collect(); + let duplicate_node = sparse.get_node_by_id(true_branch).unwrap().clone(); + nodes.push((true_branch, duplicate_node)); + let digests = sparse.digest_entries().iter().map(|(&id, &digest)| (id, digest)).collect(); + + let result = SparseMastForest::from_serialized_parts( + nodes, + digests, + sparse.num_nodes(), + sparse.procedure_roots().to_vec(), + sparse.advice_map().clone(), + sparse.commitment(), + ); + + assert_matches!( + result, + Err(DeserializationError::InvalidValue(msg)) if msg.contains("duplicate sparse full-node id") + ); +} + +#[test] +fn sparse_serialized_parts_reject_duplicate_digest_only_ids() { + let (_source, sparse, _true_branch, false_branch, _root) = sparse_split_fixture(); + let nodes = sparse.nodes().iter().map(|(&id, node)| (id, node.clone())).collect(); + let mut digests: Vec<_> = + sparse.digest_entries().iter().map(|(&id, &digest)| (id, digest)).collect(); + let digest = sparse.get_digest_by_id(false_branch).unwrap(); + digests.push((false_branch, digest)); + + let result = SparseMastForest::from_serialized_parts( + nodes, + digests, + sparse.num_nodes(), + sparse.procedure_roots().to_vec(), + sparse.advice_map().clone(), + sparse.commitment(), + ); + + assert_matches!( + result, + Err(DeserializationError::InvalidValue(msg)) if msg.contains("duplicate sparse digest-only id") + ); +} + +#[test] +fn sparse_serialized_parts_reject_full_digest_overlap() { + let (_source, sparse, true_branch, _false_branch, _root) = sparse_split_fixture(); + let nodes = sparse.nodes().iter().map(|(&id, node)| (id, node.clone())).collect(); + let mut digests: Vec<_> = + sparse.digest_entries().iter().map(|(&id, &digest)| (id, digest)).collect(); + digests.push((true_branch, sparse.get_digest_by_id(true_branch).unwrap())); + + let result = SparseMastForest::from_serialized_parts( + nodes, + digests, + sparse.num_nodes(), + sparse.procedure_roots().to_vec(), + sparse.advice_map().clone(), + sparse.commitment(), + ); + + assert_matches!( + result, + Err(DeserializationError::InvalidValue(msg)) if msg.contains("overlaps a digest-only entry") + ); +} + +#[test] +fn sparse_serialized_parts_reject_out_of_range_full_digest_and_root_ids() { + let (_source, sparse, true_branch, false_branch, _root) = sparse_split_fixture(); + let out_of_range = MastNodeId::from(sparse.num_nodes() as u32); + + let mut nodes: Vec<_> = sparse.nodes().iter().map(|(&id, node)| (id, node.clone())).collect(); + nodes.push((out_of_range, sparse.get_node_by_id(true_branch).unwrap().clone())); + let digests: Vec<_> = + sparse.digest_entries().iter().map(|(&id, &digest)| (id, digest)).collect(); + let result = SparseMastForest::from_serialized_parts( + nodes, + digests.clone(), + sparse.num_nodes(), + sparse.procedure_roots().to_vec(), + sparse.advice_map().clone(), + sparse.commitment(), + ); + assert_matches!( + result, + Err(DeserializationError::InvalidValue(msg)) if msg.contains("full node id") + && msg.contains("out of range") + ); + + let nodes = sparse.nodes().iter().map(|(&id, node)| (id, node.clone())).collect(); + let mut out_of_range_digests = digests; + out_of_range_digests.push((out_of_range, sparse.get_digest_by_id(false_branch).unwrap())); + let result = SparseMastForest::from_serialized_parts( + nodes, + out_of_range_digests, + sparse.num_nodes(), + sparse.procedure_roots().to_vec(), + sparse.advice_map().clone(), + sparse.commitment(), + ); + assert_matches!( + result, + Err(DeserializationError::InvalidValue(msg)) if msg.contains("digest-only node id") + && msg.contains("out of range") + ); + + let nodes = sparse.nodes().iter().map(|(&id, node)| (id, node.clone())).collect(); + let digests = sparse.digest_entries().iter().map(|(&id, &digest)| (id, digest)).collect(); + let result = SparseMastForest::from_serialized_parts( + nodes, + digests, + sparse.num_nodes(), + vec![out_of_range], + sparse.advice_map().clone(), + sparse.commitment(), + ); + assert_matches!( + result, + Err(DeserializationError::InvalidValue(msg)) if msg.contains("procedure root id") + && msg.contains("out of range") + ); +} + /// Test that a forest with a node whose child ids are larger than its own id serializes and /// deserializes successfully. #[test] @@ -1105,7 +1588,7 @@ fn test_batched_construction_preserves_structure() { fn assert_header_flags(bytes: &[u8], expected_flags: u8) { assert_eq!(&bytes[0..4], b"MAST", "Magic should be MAST"); assert_eq!(bytes[4], expected_flags, "unexpected serialization flags"); - assert_eq!(&bytes[5..8], &[0, 0, 4], "Version should be [0, 0, 4]"); + assert_eq!(&bytes[5..8], &[0, 0, 5], "Version should be [0, 0, 5]"); } fn read_header_counts(bytes: &[u8]) -> (usize, usize) { @@ -1247,7 +1730,7 @@ fn test_deserialize_rejects_unknown_flags() { let bytes = forest.to_bytes(); - for flag in [0x01, 0x04] { + for flag in [0x01, 0x08] { let mut bytes = bytes.clone(); bytes[4] = flag; diff --git a/core/src/mast/sparse.rs b/core/src/mast/sparse.rs index 9ceeafda46..4db9af1b95 100644 --- a/core/src/mast/sparse.rs +++ b/core/src/mast/sparse.rs @@ -10,6 +10,7 @@ use crate::{ Word, advice::AdviceMap, mast::{ExecutableMastForest, MastForest, MastNode, MastNodeExt, MastNodeId}, + utils::Idx, }; // MAST FOREST ID @@ -90,6 +91,11 @@ impl SparseMastForest { &self.advice_map } + /// Returns the digest-only entries associated with this sparse forest. + pub(in crate::mast) fn digest_entries(&self) -> &BTreeMap { + &self.digests + } + /// Returns the commitment to this sparse forest, computed from the procedure roots. /// /// The commitment value is derived from the digests of the procedure roots in the original @@ -98,6 +104,157 @@ impl SparseMastForest { pub fn commitment(&self) -> Word { self.commitment_cache } + + /// Builds a sparse forest from parts decoded from the sparse wire format. + pub(in crate::mast) fn from_serialized_parts( + nodes: Vec<(MastNodeId, MastNode)>, + digests: Vec<(MastNodeId, Word)>, + num_nodes: usize, + roots: Vec, + advice_map: AdviceMap, + commitment_cache: Word, + ) -> Result { + validate_sparse_node_bound(num_nodes)?; + + let nodes = collect_unique_nodes(nodes, num_nodes)?; + let digests = collect_unique_digests(digests, num_nodes)?; + + for &root in &roots { + validate_sparse_id(root, num_nodes, "procedure root")?; + } + + for node_id in nodes.keys() { + if digests.contains_key(node_id) { + return Err(crate::serde::DeserializationError::InvalidValue(format!( + "sparse full-node id {} overlaps a digest-only entry", + node_id.0 + ))); + } + } + + validate_full_node_child_digests(&nodes, &digests, num_nodes)?; + + Ok(Self { + nodes, + digests, + num_nodes, + roots, + advice_map, + commitment_cache, + }) + } +} + +fn validate_sparse_node_bound(num_nodes: usize) -> Result<(), crate::serde::DeserializationError> { + if num_nodes > MastForest::MAX_NODES { + return Err(crate::serde::DeserializationError::InvalidValue(format!( + "sparse source node count {num_nodes} exceeds maximum allowed {}", + MastForest::MAX_NODES + ))); + } + Ok(()) +} + +fn validate_sparse_id( + id: MastNodeId, + num_nodes: usize, + label: &str, +) -> Result<(), crate::serde::DeserializationError> { + if id.to_usize() >= num_nodes { + return Err(crate::serde::DeserializationError::InvalidValue(format!( + "{label} id {} is out of range for sparse source node count {num_nodes}", + id.0 + ))); + } + Ok(()) +} + +fn collect_unique_nodes( + nodes: Vec<(MastNodeId, MastNode)>, + num_nodes: usize, +) -> Result, crate::serde::DeserializationError> { + let mut result = BTreeMap::new(); + for (id, node) in nodes { + validate_sparse_id(id, num_nodes, "full node")?; + if result.insert(id, node).is_some() { + return Err(crate::serde::DeserializationError::InvalidValue(format!( + "duplicate sparse full-node id {}", + id.0 + ))); + } + } + Ok(result) +} + +fn collect_unique_digests( + digests: Vec<(MastNodeId, Word)>, + num_nodes: usize, +) -> Result, crate::serde::DeserializationError> { + let mut result = BTreeMap::new(); + for (id, digest) in digests { + validate_sparse_id(id, num_nodes, "digest-only node")?; + if result.insert(id, digest).is_some() { + return Err(crate::serde::DeserializationError::InvalidValue(format!( + "duplicate sparse digest-only id {}", + id.0 + ))); + } + } + Ok(result) +} + +fn validate_full_node_child_digests( + nodes: &BTreeMap, + digests: &BTreeMap, + num_nodes: usize, +) -> Result<(), crate::serde::DeserializationError> { + for (&node_id, node) in nodes { + validate_sparse_id(node_id, num_nodes, "full node")?; + + match node { + MastNode::Block(block) => { + block.validate_batch_invariants().map_err(|error_msg| { + crate::serde::DeserializationError::InvalidValue(format!( + "invalid sparse basic block {}: {error_msg}", + node_id.0 + )) + })?; + }, + MastNode::External(_) | MastNode::Dyn(_) => {}, + MastNode::Join(join) => { + require_child_digest(node_id, join.first(), nodes, digests, num_nodes)?; + require_child_digest(node_id, join.second(), nodes, digests, num_nodes)?; + }, + MastNode::Split(split) => { + require_child_digest(node_id, split.on_true(), nodes, digests, num_nodes)?; + require_child_digest(node_id, split.on_false(), nodes, digests, num_nodes)?; + }, + MastNode::Loop(loop_node) => { + require_child_digest(node_id, loop_node.body(), nodes, digests, num_nodes)?; + }, + MastNode::Call(call) => { + require_child_digest(node_id, call.callee(), nodes, digests, num_nodes)?; + }, + } + } + Ok(()) +} + +fn require_child_digest( + parent_id: MastNodeId, + child_id: MastNodeId, + nodes: &BTreeMap, + digests: &BTreeMap, + num_nodes: usize, +) -> Result<(), crate::serde::DeserializationError> { + validate_sparse_id(child_id, num_nodes, "child")?; + if !nodes.contains_key(&child_id) && !digests.contains_key(&child_id) { + return Err(crate::serde::DeserializationError::InvalidValue(format!( + "sparse full node {} references child {} without a full node or digest-only entry", + parent_id.0, child_id.0 + ))); + } + Ok(()) } impl ExecutableMastForest for SparseMastForest { From b755ad731a4e7f8ba2893cdd26d0d74e61e885b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Garillot?= Date: Tue, 30 Jun 2026 19:23:06 -0400 Subject: [PATCH 2/9] chore: Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d4127975f..0fc6a85c84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ - Reduced optimized benchmark build time by relaxing forced inlining in processor execution helpers ([#3292](https://github.com/0xMiden/miden-vm/pull/3292)). - Added no-op handlers for readonly debugger events to `CoreLibrary::handlers`, so hosts that load the core library can execute programs emitting those events without registering no-op handlers manually ([#3305](https://github.com/0xMiden/miden-vm/pull/3305)). +- Added trusted sparse MAST forest serialization for trace replay payloads ([#3313](https://github.com/0xMiden/miden-vm/pull/3313)). ## v0.24.0 (2026-06-24) From ba05f7da14e870c18739c1e27ba87c0824e5d685 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Garillot?= Date: Tue, 30 Jun 2026 20:52:58 -0400 Subject: [PATCH 3/9] fix: Separate sparse MAST replay serialization --- core/src/mast/mod.rs | 20 ++--------- core/src/mast/serialization/layout.rs | 13 +------- core/src/mast/serialization/mod.rs | 25 +++++--------- core/src/mast/serialization/sparse.rs | 48 +++++++++++++-------------- core/src/mast/serialization/tests.rs | 15 ++++----- 5 files changed, 41 insertions(+), 80 deletions(-) diff --git a/core/src/mast/mod.rs b/core/src/mast/mod.rs index 134e4ec3e2..e42dc7c24a 100644 --- a/core/src/mast/mod.rs +++ b/core/src/mast/mod.rs @@ -54,7 +54,7 @@ use proptest::prelude::*; use serde::{Deserialize, Serialize}; #[cfg(feature = "serde")] -use crate::serde::SliceReader; +use crate::serde::{Deserializable, SliceReader}; mod node; #[cfg(any(test, feature = "arbitrary"))] @@ -71,7 +71,7 @@ use crate::{ Felt, Word, advice::AdviceMap, crypto::hash::Poseidon2, - serde::{ByteWriter, Deserializable, DeserializationError, Serializable}, + serde::{ByteWriter, DeserializationError, Serializable}, utils::{DenseIdMap, Idx, IndexVec, hash_string_to_word}, }; @@ -1168,10 +1168,6 @@ impl ExecutableMastForest for Arc { #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(transparent))] -#[cfg_attr( - all(feature = "arbitrary", test), - miden_test_serde_macros::serde_test(binary_serde(true)) -)] pub struct MastNodeId(u32); /// Operations that mutate a MAST often produce this mapping between old and new NodeIds. @@ -1246,18 +1242,6 @@ impl Serializable for MastNodeId { } } -impl Deserializable for MastNodeId { - fn read_from( - source: &mut R, - ) -> Result { - Ok(Self(::read_from(source)?)) - } - - fn min_serialized_size() -> usize { - ::min_serialized_size() - } -} - impl fmt::Display for MastNodeId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "MastNodeId({})", self.0) diff --git a/core/src/mast/serialization/layout.rs b/core/src/mast/serialization/layout.rs index b790e593b1..6b8a592641 100644 --- a/core/src/mast/serialization/layout.rs +++ b/core/src/mast/serialization/layout.rs @@ -1,8 +1,6 @@ use alloc::{format, string::ToString, vec::Vec}; -use super::{ - FLAG_HASHLESS, FLAG_SPARSE, FLAGS_RESERVED_MASK, MAGIC, MastForest, MastNodeEntry, VERSION, -}; +use super::{FLAG_HASHLESS, FLAGS_RESERVED_MASK, MAGIC, MastForest, MastNodeEntry, VERSION}; use crate::{ mast::MastNodeId, serde::{ByteReader, Deserializable, DeserializationError, SliceReader}, @@ -159,10 +157,6 @@ impl WireFlags { pub(super) fn is_hashless(self) -> bool { self.0 & FLAG_HASHLESS != 0 } - - pub(super) fn is_sparse(self) -> bool { - self.0 & FLAG_SPARSE != 0 - } } // LAYOUT SCANNING @@ -177,11 +171,6 @@ pub(super) fn read_header_and_scan_layout( // untrusted deserialization path. let (raw_flags, _version) = read_and_validate_header(source)?; let flags = WireFlags::new(raw_flags); - if flags.is_sparse() { - return Err(DeserializationError::InvalidValue( - "SPARSE flag is set; use SparseMastForest for sparse replay input".to_string(), - )); - } if flags.is_hashless() && !allow_hashless { return Err(DeserializationError::InvalidValue( "HASHLESS flag is set; use UntrustedMastForest for untrusted input".to_string(), diff --git a/core/src/mast/serialization/mod.rs b/core/src/mast/serialization/mod.rs index e70690164d..be61542fbe 100644 --- a/core/src/mast/serialization/mod.rs +++ b/core/src/mast/serialization/mod.rs @@ -76,14 +76,13 @@ //! same contiguous array on the wire. //! //! Public entry points adopt these policies: -//! - [`MastForest::read_from_bytes`]: trusted dense execution payload, no hashless or sparse -//! support. +//! - [`MastForest::read_from_bytes`]: trusted dense execution payload, no hashless support. //! - [`MastForestWireView::new`]: trusted wire-backed cache access; rejects hashless and legacy -//! debug-bearing payloads, and rejects sparse payloads. +//! debug-bearing payloads. //! - [`crate::mast::SparseMastForest::read_from_bytes`] / -//! [`crate::mast::SparseMastForest::read_from_bytes_with_options`]: trusted sparse replay -//! payloads for serialized trace-generation inputs. Sparse payloads currently carry full-node -//! digests and do not recompute them on read. +//! [`crate::mast::SparseMastForest::read_from_bytes_with_options`]: separate trusted sparse +//! replay payloads for serialized trace-generation inputs. Sparse payloads currently carry +//! full-node digests and do not recompute them on read. //! - [`crate::mast::UntrustedMastForest::read_from_bytes`] / //! [`crate::mast::UntrustedMastForest::read_from_bytes_with_options`]: untrusted parsing plus //! later validation before use. @@ -179,16 +178,10 @@ const MAGIC: &[u8; 4] = b"MAST"; /// from local structure. pub(super) const FLAG_HASHLESS: u8 = 0x02; -/// Flag indicating that the payload uses sparse MAST replay serialization. -/// -/// Sparse payloads preserve the source forest's [`MastNodeId`] space and therefore cannot be read -/// through dense [`MastForest`] entry points. -pub(super) const FLAG_SPARSE: u8 = 0x04; - /// Mask for reserved flag bits that must be zero. /// -/// Bit 0 and bits 3-7 are reserved for future use. If any are set, deserialization fails. -const FLAGS_RESERVED_MASK: u8 = 0xf9; +/// Bit 0 and bits 2-7 are reserved for future use. If any are set, deserialization fails. +const FLAGS_RESERVED_MASK: u8 = 0xfd; /// The format version. /// @@ -215,15 +208,13 @@ const FLAGS_RESERVED_MASK: u8 = 0xf9; /// records. MAST nodes are metadata-free identifiers. Before any public release on this branch, /// the same unreleased wire version also reserved bit 0 and stopped using it as a forest-level /// debug-presence flag. -/// - [0, 0, 5]: Added SPARSE flag (bit 2). Sparse payloads preserve sparse replay IDs and are -/// accepted only by SparseMastForest readers. /// /// Legacy wire versions (pre-#3192 decorator terminology): /// [0,0,1] stored metadata as serialized decorator variants in CSR per-node slots. /// [0,0,2] removed AssemblyOp from the decorator enum and stored them separately in DebugInfo. /// [0,0,3] removed the unused decorator-count wire field. /// [0,0,4] eliminated the decorator wire slots entirely. -const VERSION: [u8; 3] = [0, 0, 5]; +const VERSION: [u8; 3] = [0, 0, 4]; // MAST FOREST SERIALIZATION/DESERIALIZATION // ================================================================================================ diff --git a/core/src/mast/serialization/sparse.rs b/core/src/mast/serialization/sparse.rs index 63d032061a..48ae99fc06 100644 --- a/core/src/mast/serialization/sparse.rs +++ b/core/src/mast/serialization/sparse.rs @@ -1,9 +1,9 @@ use alloc::{format, string::ToString, vec::Vec}; use super::{ - FLAG_HASHLESS, FLAG_SPARSE, MAGIC, MastNodeEntry, VERSION, + MastNodeEntry, basic_blocks::{BasicBlockDataBuilder, BasicBlockDataDecoder}, - layout::{OffsetTrackingReader, TrackingReader, WireFlags, read_and_validate_header}, + layout::{OffsetTrackingReader, TrackingReader}, }; use crate::{ Word, @@ -15,12 +15,12 @@ use crate::{ }, }; -const SPARSE_FLAGS: u8 = FLAG_HASHLESS | FLAG_SPARSE; +const SPARSE_MAGIC: &[u8; 4] = b"SMST"; +const SPARSE_VERSION: [u8; 3] = [0, 0, 0]; fn sparse_mast_forest_min_serialized_size() -> usize { - MAGIC.len() - + 1 - + VERSION.len() + SPARSE_MAGIC.len() + + SPARSE_VERSION.len() + usize::min_serialized_size() * 7 + Word::min_serialized_size() + usize::min_serialized_size() @@ -56,9 +56,8 @@ pub(super) fn write_sparse_into(forest: &SparseMastForest, target let non_external_count = entries.iter().filter(|entry| !matches!(entry, MastNodeEntry::External)).count(); - target.write_bytes(MAGIC); - target.write_u8(SPARSE_FLAGS); - target.write_bytes(&VERSION); + target.write_bytes(SPARSE_MAGIC); + target.write_bytes(&SPARSE_VERSION); target.write_usize(forest.procedure_roots().len()); target.write_usize(forest.num_nodes()); @@ -128,9 +127,7 @@ pub(super) fn read_sparse_from( source: &mut R, ) -> Result { let mut reader = TrackingReader::new(source); - let (raw_flags, _version) = read_and_validate_header(&mut reader)?; - let flags = WireFlags::new(raw_flags); - validate_sparse_flags(flags)?; + read_and_validate_sparse_header(&mut reader)?; let root_count = read_bounded_count(&mut reader, size_of::(), "procedure root count")?; let source_node_count = reader.read_usize()?; @@ -225,23 +222,24 @@ pub(super) fn read_sparse_from( ) } -fn validate_sparse_flags(flags: WireFlags) -> Result<(), DeserializationError> { - if !flags.is_sparse() { - return Err(DeserializationError::InvalidValue( - "SPARSE flag is not set; use MastForest readers for dense input".to_string(), - )); - } - if !flags.is_hashless() { - return Err(DeserializationError::InvalidValue( - "sparse MAST payloads must also set HASHLESS".to_string(), - )); +fn read_and_validate_sparse_header( + source: &mut R, +) -> Result<(), DeserializationError> { + let magic: [u8; 4] = source.read_array()?; + if magic != *SPARSE_MAGIC { + return Err(DeserializationError::InvalidValue(format!( + "Invalid sparse MAST magic bytes. Expected '{:?}', got '{:?}'", + *SPARSE_MAGIC, magic + ))); } - if flags.bits() != SPARSE_FLAGS { + + let version: [u8; 3] = source.read_array()?; + if version != SPARSE_VERSION { return Err(DeserializationError::InvalidValue(format!( - "invalid sparse MAST flag combination: {:#04x}", - flags.bits() + "Unsupported sparse MAST version. Got '{version:?}', but only '{SPARSE_VERSION:?}' is supported", ))); } + Ok(()) } diff --git a/core/src/mast/serialization/tests.rs b/core/src/mast/serialization/tests.rs index 2e2eb88266..7299cd64f5 100644 --- a/core/src/mast/serialization/tests.rs +++ b/core/src/mast/serialization/tests.rs @@ -836,9 +836,8 @@ fn write_sparse_test_payload( assert_eq!(full_ids.len(), full_digests.len()); let mut bytes = Vec::new(); - bytes.write_bytes(MAGIC); - bytes.write_u8(FLAG_HASHLESS | FLAG_SPARSE); - bytes.write_bytes(&VERSION); + bytes.write_bytes(b"SMST"); + bytes.write_bytes(&[0, 0, 0]); bytes.write_usize(roots.len()); bytes.write_usize(source_node_count); @@ -1046,19 +1045,19 @@ fn dense_mast_readers_reject_sparse_payloads() { let materialized = MastForest::read_from_bytes(&bytes); assert_matches!( materialized, - Err(DeserializationError::InvalidValue(msg)) if msg.contains("SPARSE flag is set") + Err(DeserializationError::InvalidValue(msg)) if msg.contains("Invalid magic bytes") ); let wire_view = MastForestWireView::new(&bytes); assert_matches!( wire_view, - Err(DeserializationError::InvalidValue(msg)) if msg.contains("SPARSE flag is set") + Err(DeserializationError::InvalidValue(msg)) if msg.contains("Invalid magic bytes") ); let untrusted = UntrustedMastForest::read_from_bytes(&bytes); assert_matches!( untrusted, - Err(DeserializationError::InvalidValue(msg)) if msg.contains("SPARSE flag is set") + Err(DeserializationError::InvalidValue(msg)) if msg.contains("Invalid magic bytes") ); } @@ -1073,7 +1072,7 @@ fn sparse_reader_rejects_dense_payloads() { let result = SparseMastForest::read_from_bytes(&forest.to_bytes()); assert_matches!( result, - Err(DeserializationError::InvalidValue(msg)) if msg.contains("SPARSE flag is not set") + Err(DeserializationError::InvalidValue(msg)) if msg.contains("Invalid sparse MAST magic bytes") ); } @@ -1588,7 +1587,7 @@ fn test_batched_construction_preserves_structure() { fn assert_header_flags(bytes: &[u8], expected_flags: u8) { assert_eq!(&bytes[0..4], b"MAST", "Magic should be MAST"); assert_eq!(bytes[4], expected_flags, "unexpected serialization flags"); - assert_eq!(&bytes[5..8], &[0, 0, 5], "Version should be [0, 0, 5]"); + assert_eq!(&bytes[5..8], &[0, 0, 4], "Version should be [0, 0, 4]"); } fn read_header_counts(bytes: &[u8]) -> (usize, usize) { From c6b6efa3a786739fce232987817053570a577829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Garillot?= Date: Fri, 3 Jul 2026 19:40:07 -0400 Subject: [PATCH 4/9] fix: Address sparse MAST review comments --- core/src/mast/serialization/sparse.rs | 22 +++++++++++++-- core/src/mast/sparse.rs | 39 ++++++++++++++------------- 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/core/src/mast/serialization/sparse.rs b/core/src/mast/serialization/sparse.rs index 48ae99fc06..cc8bbc0316 100644 --- a/core/src/mast/serialization/sparse.rs +++ b/core/src/mast/serialization/sparse.rs @@ -15,9 +15,15 @@ use crate::{ }, }; +// CONSTANTS +// ================================================================================================ + const SPARSE_MAGIC: &[u8; 4] = b"SMST"; const SPARSE_VERSION: [u8; 3] = [0, 0, 0]; +// HELPERS +// ================================================================================================ + fn sparse_mast_forest_min_serialized_size() -> usize { SPARSE_MAGIC.len() + SPARSE_VERSION.len() @@ -26,13 +32,16 @@ fn sparse_mast_forest_min_serialized_size() -> usize { + usize::min_serialized_size() } +// WRITER +// ================================================================================================ + /// Serializes a [`SparseMastForest`] in trusted sparse replay form. /// /// This format carries the digest for each full node and accepts those digests on read. It is /// suitable for trusted remote proving inputs, not as an untrusted hashless validation path. /// /// See for the planned untrusted reader. -pub(super) fn write_sparse_into(forest: &SparseMastForest, target: &mut W) { +fn write_sparse_into(forest: &SparseMastForest, target: &mut W) { let mut basic_block_data_builder = BasicBlockDataBuilder::new(); let mut full_ids = Vec::with_capacity(forest.nodes().len()); let mut entries = Vec::with_capacity(forest.nodes().len()); @@ -94,6 +103,9 @@ pub(super) fn write_sparse_into(forest: &SparseMastForest, target forest.advice_map().write_into(target); } +// TRAIT IMPLS +// ================================================================================================ + impl Serializable for SparseMastForest { fn write_into(&self, target: &mut W) { write_sparse_into(self, target); @@ -119,11 +131,14 @@ impl Deserializable for SparseMastForest { } } +// READER +// ================================================================================================ + /// Reads a trusted sparse replay payload. /// /// The payload carries full-node digests and digest-only entries as replay data. It does not /// rebuild those hashes from node structure. -pub(super) fn read_sparse_from( +fn read_sparse_from( source: &mut R, ) -> Result { let mut reader = TrackingReader::new(source); @@ -222,6 +237,9 @@ pub(super) fn read_sparse_from( ) } +// HELPER FUNCTIONS +// ================================================================================================ + fn read_and_validate_sparse_header( source: &mut R, ) -> Result<(), DeserializationError> { diff --git a/core/src/mast/sparse.rs b/core/src/mast/sparse.rs index 4db9af1b95..6ff7d6cc05 100644 --- a/core/src/mast/sparse.rs +++ b/core/src/mast/sparse.rs @@ -10,6 +10,7 @@ use crate::{ Word, advice::AdviceMap, mast::{ExecutableMastForest, MastForest, MastNode, MastNodeExt, MastNodeId}, + serde::DeserializationError, utils::Idx, }; @@ -51,10 +52,10 @@ pub struct SparseMastForest { /// full-node entry implicitly carries its own digest via [`MastNodeExt::digest`]. digests: BTreeMap, - /// Total number of nodes in the source [`MastForest`] from which this sparse forest was - /// built. Note that this is *not* `nodes.len()` — it is the upper bound on the original - /// [`MastNodeId`] space, preserved so that callers materializing dense-shaped state (e.g. - /// allocating an `IndexVec` keyed by [`MastNodeId`]) know its required size. + /// Upper bound for [`MastNodeId`] values from the source [`MastForest`]. + /// + /// This is not `nodes.len()`. The trusted reader uses it to reject IDs outside the source + /// forest's ID space, not as an independent proof of the source forest's size. num_nodes: usize, /// Roots of procedures defined within the original MAST forest. @@ -113,7 +114,7 @@ impl SparseMastForest { roots: Vec, advice_map: AdviceMap, commitment_cache: Word, - ) -> Result { + ) -> Result { validate_sparse_node_bound(num_nodes)?; let nodes = collect_unique_nodes(nodes, num_nodes)?; @@ -125,7 +126,7 @@ impl SparseMastForest { for node_id in nodes.keys() { if digests.contains_key(node_id) { - return Err(crate::serde::DeserializationError::InvalidValue(format!( + return Err(DeserializationError::InvalidValue(format!( "sparse full-node id {} overlaps a digest-only entry", node_id.0 ))); @@ -145,9 +146,9 @@ impl SparseMastForest { } } -fn validate_sparse_node_bound(num_nodes: usize) -> Result<(), crate::serde::DeserializationError> { +fn validate_sparse_node_bound(num_nodes: usize) -> Result<(), DeserializationError> { if num_nodes > MastForest::MAX_NODES { - return Err(crate::serde::DeserializationError::InvalidValue(format!( + return Err(DeserializationError::InvalidValue(format!( "sparse source node count {num_nodes} exceeds maximum allowed {}", MastForest::MAX_NODES ))); @@ -159,9 +160,9 @@ fn validate_sparse_id( id: MastNodeId, num_nodes: usize, label: &str, -) -> Result<(), crate::serde::DeserializationError> { +) -> Result<(), DeserializationError> { if id.to_usize() >= num_nodes { - return Err(crate::serde::DeserializationError::InvalidValue(format!( + return Err(DeserializationError::InvalidValue(format!( "{label} id {} is out of range for sparse source node count {num_nodes}", id.0 ))); @@ -172,12 +173,12 @@ fn validate_sparse_id( fn collect_unique_nodes( nodes: Vec<(MastNodeId, MastNode)>, num_nodes: usize, -) -> Result, crate::serde::DeserializationError> { +) -> Result, DeserializationError> { let mut result = BTreeMap::new(); for (id, node) in nodes { validate_sparse_id(id, num_nodes, "full node")?; if result.insert(id, node).is_some() { - return Err(crate::serde::DeserializationError::InvalidValue(format!( + return Err(DeserializationError::InvalidValue(format!( "duplicate sparse full-node id {}", id.0 ))); @@ -189,12 +190,12 @@ fn collect_unique_nodes( fn collect_unique_digests( digests: Vec<(MastNodeId, Word)>, num_nodes: usize, -) -> Result, crate::serde::DeserializationError> { +) -> Result, DeserializationError> { let mut result = BTreeMap::new(); for (id, digest) in digests { validate_sparse_id(id, num_nodes, "digest-only node")?; if result.insert(id, digest).is_some() { - return Err(crate::serde::DeserializationError::InvalidValue(format!( + return Err(DeserializationError::InvalidValue(format!( "duplicate sparse digest-only id {}", id.0 ))); @@ -203,18 +204,20 @@ fn collect_unique_digests( Ok(result) } +/// Checks that every child of a retained full node is available as either a full node or a +/// digest-only entry. fn validate_full_node_child_digests( nodes: &BTreeMap, digests: &BTreeMap, num_nodes: usize, -) -> Result<(), crate::serde::DeserializationError> { +) -> Result<(), DeserializationError> { for (&node_id, node) in nodes { validate_sparse_id(node_id, num_nodes, "full node")?; match node { MastNode::Block(block) => { block.validate_batch_invariants().map_err(|error_msg| { - crate::serde::DeserializationError::InvalidValue(format!( + DeserializationError::InvalidValue(format!( "invalid sparse basic block {}: {error_msg}", node_id.0 )) @@ -246,10 +249,10 @@ fn require_child_digest( nodes: &BTreeMap, digests: &BTreeMap, num_nodes: usize, -) -> Result<(), crate::serde::DeserializationError> { +) -> Result<(), DeserializationError> { validate_sparse_id(child_id, num_nodes, "child")?; if !nodes.contains_key(&child_id) && !digests.contains_key(&child_id) { - return Err(crate::serde::DeserializationError::InvalidValue(format!( + return Err(DeserializationError::InvalidValue(format!( "sparse full node {} references child {} without a full node or digest-only entry", parent_id.0, child_id.0 ))); From 280baa4ec5bedcc13512e63701e153daf06b20f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Garillot?= Date: Fri, 10 Jul 2026 13:05:48 -0400 Subject: [PATCH 5/9] refactor: simplify sparse MAST trusted serialization --- core/src/mast/mod.rs | 18 +- core/src/mast/serialization/layout.rs | 2 +- core/src/mast/serialization/mod.rs | 9 +- core/src/mast/serialization/sparse.rs | 554 +++++++++++--------------- core/src/mast/serialization/tests.rs | 360 ++++++----------- core/src/mast/sparse.rs | 17 +- 6 files changed, 386 insertions(+), 574 deletions(-) diff --git a/core/src/mast/mod.rs b/core/src/mast/mod.rs index e42dc7c24a..abd401d3d1 100644 --- a/core/src/mast/mod.rs +++ b/core/src/mast/mod.rs @@ -54,7 +54,7 @@ use proptest::prelude::*; use serde::{Deserialize, Serialize}; #[cfg(feature = "serde")] -use crate::serde::{Deserializable, SliceReader}; +use crate::serde::SliceReader; mod node; #[cfg(any(test, feature = "arbitrary"))] @@ -71,14 +71,14 @@ use crate::{ Felt, Word, advice::AdviceMap, crypto::hash::Poseidon2, - serde::{ByteWriter, DeserializationError, Serializable}, + serde::{ByteWriter, Deserializable, DeserializationError, Serializable}, utils::{DenseIdMap, Idx, IndexVec, hash_string_to_word}, }; mod serialization; pub use serialization::{ AdviceMapView, AdviceValueView, MastForestReadMode, MastForestReadView, MastForestView, - MastForestWireView, MastNodeEntry, MastNodeInfo, SparseMastForestReadOptions, + MastForestWireView, MastNodeEntry, MastNodeInfo, }; mod dense_builder; @@ -1242,6 +1242,18 @@ impl Serializable for MastNodeId { } } +impl Deserializable for MastNodeId { + fn read_from( + source: &mut R, + ) -> Result { + Ok(Self(::read_from(source)?)) + } + + fn min_serialized_size() -> usize { + ::min_serialized_size() + } +} + impl fmt::Display for MastNodeId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "MastNodeId({})", self.0) diff --git a/core/src/mast/serialization/layout.rs b/core/src/mast/serialization/layout.rs index 6b8a592641..95fab0d2ca 100644 --- a/core/src/mast/serialization/layout.rs +++ b/core/src/mast/serialization/layout.rs @@ -457,7 +457,7 @@ fn validate_budgeted_count( Ok(()) } -pub(super) fn read_and_validate_header( +fn read_and_validate_header( source: &mut R, ) -> Result<(u8, [u8; 3]), DeserializationError> { let magic: [u8; 4] = source.read_array()?; diff --git a/core/src/mast/serialization/mod.rs b/core/src/mast/serialization/mod.rs index be61542fbe..972ca9b895 100644 --- a/core/src/mast/serialization/mod.rs +++ b/core/src/mast/serialization/mod.rs @@ -79,10 +79,10 @@ //! - [`MastForest::read_from_bytes`]: trusted dense execution payload, no hashless support. //! - [`MastForestWireView::new`]: trusted wire-backed cache access; rejects hashless and legacy //! debug-bearing payloads. -//! - [`crate::mast::SparseMastForest::read_from_bytes`] / -//! [`crate::mast::SparseMastForest::read_from_bytes_with_options`]: separate trusted sparse -//! replay payloads for serialized trace-generation inputs. Sparse payloads currently carry -//! full-node digests and do not recompute them on read. +//! - [`crate::mast::SparseMastForest::read_from_bytes`]: separate trusted sparse replay payloads +//! for serialized trace-generation inputs. Sparse payloads preserve the sparse node and digest +//! maps produced by tracing; they do not share the dense `MastForest` wire format and are not an +//! untrusted validation boundary. //! - [`crate::mast::UntrustedMastForest::read_from_bytes`] / //! [`crate::mast::UntrustedMastForest::read_from_bytes_with_options`]: untrusted parsing plus //! later validation before use. @@ -117,7 +117,6 @@ pub(super) use layout::ForestLayout; use layout::{OffsetTrackingReader, TrackingReader, WireFlags, read_header_and_scan_layout}; mod sparse; -pub use sparse::SparseMastForestReadOptions; mod resolved; use resolved::{ResolvedSerializedForest, basic_block_offset_for_node_index}; diff --git a/core/src/mast/serialization/sparse.rs b/core/src/mast/serialization/sparse.rs index cc8bbc0316..a4277153cd 100644 --- a/core/src/mast/serialization/sparse.rs +++ b/core/src/mast/serialization/sparse.rs @@ -1,106 +1,117 @@ -use alloc::{format, string::ToString, vec::Vec}; +use alloc::{collections::BTreeMap, format, string::ToString, vec::Vec}; -use super::{ - MastNodeEntry, - basic_blocks::{BasicBlockDataBuilder, BasicBlockDataDecoder}, - layout::{OffsetTrackingReader, TrackingReader}, -}; +use super::basic_blocks::{BasicBlockDataBuilder, BasicBlockDataDecoder}; use crate::{ Word, advice::AdviceMap, - mast::{MastForest, MastNode, MastNodeExt, MastNodeId, SparseMastForest}, + mast::{ + BasicBlockNodeBuilder, CallNodeBuilder, DynNodeBuilder, ExternalNodeBuilder, + JoinNodeBuilder, LoopNodeBuilder, MastForestContributor, MastNode, MastNodeExt, MastNodeId, + SparseMastForest, SplitNodeBuilder, + }, serde::{ - BudgetedReader, ByteReader, ByteWriter, Deserializable, DeserializationError, Serializable, - SliceReader, + ByteReader, ByteWriter, Deserializable, DeserializationError, Serializable, SliceReader, }, }; -// CONSTANTS -// ================================================================================================ - -const SPARSE_MAGIC: &[u8; 4] = b"SMST"; -const SPARSE_VERSION: [u8; 3] = [0, 0, 0]; - -// HELPERS -// ================================================================================================ - -fn sparse_mast_forest_min_serialized_size() -> usize { - SPARSE_MAGIC.len() - + SPARSE_VERSION.len() - + usize::min_serialized_size() * 7 - + Word::min_serialized_size() - + usize::min_serialized_size() -} +const SPARSE_BLOCK: u8 = 0; +const SPARSE_JOIN: u8 = 1; +const SPARSE_SPLIT: u8 = 2; +const SPARSE_LOOP: u8 = 3; +const SPARSE_CALL: u8 = 4; +const SPARSE_SYSCALL: u8 = 5; +const SPARSE_DYN: u8 = 6; +const SPARSE_DYNCALL: u8 = 7; +const SPARSE_EXTERNAL: u8 = 8; // WRITER // ================================================================================================ -/// Serializes a [`SparseMastForest`] in trusted sparse replay form. +/// Writes trusted sparse trace replay data. /// -/// This format carries the digest for each full node and accepts those digests on read. It is -/// suitable for trusted remote proving inputs, not as an untrusted hashless validation path. -/// -/// See for the planned untrusted reader. +/// This format preserves the sparse maps produced by execution tracing. It does not prove that +/// this sparse view is a subset of a committed [`MastForest`], and it does not share the dense +/// [`MastForest`] wire format. Callers must only read these bytes from a trusted producer, or after +/// an outer transport/authentication layer has accepted them. fn write_sparse_into(forest: &SparseMastForest, target: &mut W) { - let mut basic_block_data_builder = BasicBlockDataBuilder::new(); - let mut full_ids = Vec::with_capacity(forest.nodes().len()); - let mut entries = Vec::with_capacity(forest.nodes().len()); - let mut full_digests = Vec::with_capacity(forest.nodes().len()); - - for (&node_id, node) in forest.nodes() { - let ops_offset = if let MastNode::Block(basic_block) = node { - basic_block_data_builder.encode_basic_block(basic_block) - } else { - 0 - }; - - full_ids.push(node_id); - entries.push(MastNodeEntry::new(node, ops_offset)); - full_digests.push(node.digest()); - } - - let basic_block_data = basic_block_data_builder.finalize(); - let external_full_node_count = - entries.iter().filter(|entry| matches!(entry, MastNodeEntry::External)).count(); - let non_external_count = - entries.iter().filter(|entry| !matches!(entry, MastNodeEntry::External)).count(); - - target.write_bytes(SPARSE_MAGIC); - target.write_bytes(&SPARSE_VERSION); - - target.write_usize(forest.procedure_roots().len()); - target.write_usize(forest.num_nodes()); - target.write_usize(full_ids.len()); - target.write_usize(forest.digest_entries().len()); - target.write_usize(external_full_node_count); - target.write_usize(non_external_count); - target.write_usize(basic_block_data.len()); - - for &root in forest.procedure_roots() { - root.0.write_into(target); - } - - forest.commitment().write_into(target); - target.write_bytes(&basic_block_data); + forest.num_nodes().write_into(target); + write_node_ids(forest.procedure_roots(), target); + write_sparse_nodes(forest.nodes(), target); + write_digest_entries(forest.digest_entries(), target); + forest.advice_map().write_into(target); +} - for id in full_ids { - id.0.write_into(target); +fn write_node_ids(ids: &[MastNodeId], target: &mut W) { + target.write_usize(ids.len()); + for id in ids { + id.write_into(target); } +} - for entry in entries { - entry.write_into(target); +fn write_sparse_nodes(nodes: &BTreeMap, target: &mut W) { + target.write_usize(nodes.len()); + for (&id, node) in nodes { + id.write_into(target); + write_sparse_node(node, target); } +} - for digest in full_digests { +fn write_digest_entries(digests: &BTreeMap, target: &mut W) { + target.write_usize(digests.len()); + for (&id, &digest) in digests { + id.write_into(target); digest.write_into(target); } +} - for (&id, &digest) in forest.digest_entries() { - id.0.write_into(target); - digest.write_into(target); +fn write_sparse_node(node: &MastNode, target: &mut W) { + match node { + MastNode::Block(block) => { + target.write_u8(SPARSE_BLOCK); + node.digest().write_into(target); + + let mut basic_block_data = BasicBlockDataBuilder::new(); + let ops_offset = basic_block_data.encode_basic_block(block); + debug_assert_eq!(ops_offset, 0); + let basic_block_data = basic_block_data.finalize(); + target.write_usize(basic_block_data.len()); + target.write_bytes(&basic_block_data); + }, + MastNode::Join(join) => { + target.write_u8(SPARSE_JOIN); + node.digest().write_into(target); + join.first().write_into(target); + join.second().write_into(target); + }, + MastNode::Split(split) => { + target.write_u8(SPARSE_SPLIT); + node.digest().write_into(target); + split.on_true().write_into(target); + split.on_false().write_into(target); + }, + MastNode::Loop(loop_node) => { + target.write_u8(SPARSE_LOOP); + node.digest().write_into(target); + loop_node.body().write_into(target); + }, + MastNode::Call(call) => { + target.write_u8(if call.is_syscall() { SPARSE_SYSCALL } else { SPARSE_CALL }); + node.digest().write_into(target); + call.callee().write_into(target); + }, + MastNode::Dyn(dyn_node) => { + target.write_u8(if dyn_node.is_dyncall() { + SPARSE_DYNCALL + } else { + SPARSE_DYN + }); + node.digest().write_into(target); + }, + MastNode::External(_) => { + target.write_u8(SPARSE_EXTERNAL); + node.digest().write_into(target); + }, } - - forest.advice_map().write_into(target); } // TRAIT IMPLS @@ -113,119 +124,43 @@ impl Serializable for SparseMastForest { } impl Deserializable for SparseMastForest { - /// Reads a trusted sparse replay payload. - /// - /// Full-node digests are accepted from the payload. This is not the untrusted hash-validation - /// path from . fn read_from(source: &mut R) -> Result { read_sparse_from(source) } fn min_serialized_size() -> usize { - sparse_mast_forest_min_serialized_size() + usize::min_serialized_size() } - /// Reads trusted sparse replay bytes and rejects trailing bytes. + /// Reads one trusted sparse replay payload and rejects trailing bytes. + /// + /// This is not an untrusted input format. The reader performs cheap structural checks, but a + /// producer controls collection lengths and can drive allocation. Callers must only read these + /// bytes from a trusted producer, or after an outer transport/authentication layer has accepted + /// them. fn read_from_bytes(bytes: &[u8]) -> Result { - SparseMastForest::read_from_bytes(bytes) + let mut reader = SliceReader::new(bytes); + let forest = read_sparse_from(&mut reader)?; + if reader.has_more_bytes() { + return Err(DeserializationError::InvalidValue( + "extra bytes after SparseMastForest payload".to_string(), + )); + } + Ok(forest) } } // READER // ================================================================================================ -/// Reads a trusted sparse replay payload. -/// -/// The payload carries full-node digests and digest-only entries as replay data. It does not -/// rebuild those hashes from node structure. fn read_sparse_from( source: &mut R, ) -> Result { - let mut reader = TrackingReader::new(source); - read_and_validate_sparse_header(&mut reader)?; - - let root_count = read_bounded_count(&mut reader, size_of::(), "procedure root count")?; - let source_node_count = reader.read_usize()?; - if source_node_count > MastForest::MAX_NODES { - return Err(DeserializationError::InvalidValue(format!( - "source node count {source_node_count} exceeds maximum allowed {}", - MastForest::MAX_NODES - ))); - } - - let full_node_count = read_bounded_count( - &mut reader, - MastNodeEntry::SERIALIZED_SIZE + Word::min_serialized_size(), - "full node count", - )?; - let digest_only_count = read_bounded_count( - &mut reader, - size_of::() + Word::min_serialized_size(), - "digest-only node count", - )?; - let external_full_node_count = read_bounded_count( - &mut reader, - MastNodeEntry::SERIALIZED_SIZE, - "external full-node count", - )?; - let non_external_full_node_count = read_bounded_count( - &mut reader, - MastNodeEntry::SERIALIZED_SIZE, - "non-external full-node count", - )?; - let basic_block_data_len = read_bounded_count(&mut reader, 1, "basic-block data length")?; - - let counted_full = external_full_node_count - .checked_add(non_external_full_node_count) - .ok_or_else(|| { - DeserializationError::InvalidValue("full node count overflow".to_string()) - })?; - if counted_full != full_node_count { - return Err(DeserializationError::InvalidValue(format!( - "sparse header full node count {full_node_count} does not match external + non-external count {counted_full}" - ))); - } - - let roots = read_id_section(&mut reader, root_count, source_node_count, "procedure root")?; - let commitment = Word::read_from(&mut reader)?; - let basic_block_data = reader.read_slice(basic_block_data_len)?.to_vec(); - let full_ids = read_id_section(&mut reader, full_node_count, source_node_count, "full node")?; - validate_strictly_increasing_ids(&full_ids, "full node")?; - - let mut entries = Vec::with_capacity(full_node_count); - for _ in 0..full_node_count { - entries.push(MastNodeEntry::read_from(&mut reader)?); - } - - let counted_external = - entries.iter().filter(|entry| matches!(entry, MastNodeEntry::External)).count(); - if counted_external != external_full_node_count { - return Err(DeserializationError::InvalidValue(format!( - "sparse header external full-node count {external_full_node_count} does not match {counted_external} external entries" - ))); - } - - let mut full_digests = Vec::with_capacity(full_node_count); - for _ in 0..full_node_count { - full_digests.push(Word::read_from(&mut reader)?); - } - - let mut digest_entries = Vec::with_capacity(digest_only_count); - for _ in 0..digest_only_count { - let id = read_node_id(&mut reader, source_node_count, "digest-only node")?; - let digest = Word::read_from(&mut reader)?; - digest_entries.push((id, digest)); - } - validate_strictly_increasing_entry_ids(&digest_entries, "digest-only node")?; - - let advice_map = AdviceMap::read_from(&mut reader)?; - let nodes = materialize_sparse_nodes( - &full_ids, - &entries, - &full_digests, - source_node_count, - &basic_block_data, - )?; + let source_node_count = source.read_usize()?; + let roots = read_node_ids(source, source_node_count, "procedure root")?; + let nodes = read_sparse_nodes(source, source_node_count)?; + let digest_entries = read_digest_entries(source, source_node_count)?; + let advice_map = AdviceMap::read_from(source)?; SparseMastForest::from_serialized_parts( nodes, @@ -233,35 +168,133 @@ fn read_sparse_from( source_node_count, roots, advice_map, - commitment, ) } -// HELPER FUNCTIONS -// ================================================================================================ +fn read_node_ids( + source: &mut R, + source_node_count: usize, + label: &str, +) -> Result, DeserializationError> { + let count = read_bounded_count(source, u32::min_serialized_size(), label)?; + let mut ids = Vec::with_capacity(count); + for _ in 0..count { + ids.push(read_node_id(source, source_node_count, label)?); + } + Ok(ids) +} -fn read_and_validate_sparse_header( +fn read_sparse_nodes( source: &mut R, -) -> Result<(), DeserializationError> { - let magic: [u8; 4] = source.read_array()?; - if magic != *SPARSE_MAGIC { - return Err(DeserializationError::InvalidValue(format!( - "Invalid sparse MAST magic bytes. Expected '{:?}', got '{:?}'", - *SPARSE_MAGIC, magic - ))); + source_node_count: usize, +) -> Result, DeserializationError> { + let count = read_bounded_count(source, sparse_node_min_size(), "full node count")?; + let mut nodes = Vec::with_capacity(count); + let mut previous_id = None; + + for _ in 0..count { + let id = read_node_id(source, source_node_count, "full node")?; + validate_strictly_increasing_id(previous_id, id, "full node")?; + let node = read_sparse_node(source, source_node_count, id)?; + nodes.push((id, node)); + previous_id = Some(id); } - let version: [u8; 3] = source.read_array()?; - if version != SPARSE_VERSION { - return Err(DeserializationError::InvalidValue(format!( - "Unsupported sparse MAST version. Got '{version:?}', but only '{SPARSE_VERSION:?}' is supported", - ))); + Ok(nodes) +} + +fn read_digest_entries( + source: &mut R, + source_node_count: usize, +) -> Result, DeserializationError> { + let count = + read_bounded_count(source, sparse_digest_entry_min_size(), "digest-only node count")?; + let mut digests = Vec::with_capacity(count); + let mut previous_id = None; + + for _ in 0..count { + let id = read_node_id(source, source_node_count, "digest-only node")?; + validate_strictly_increasing_id(previous_id, id, "digest-only node")?; + let digest = Word::read_from(source)?; + digests.push((id, digest)); + previous_id = Some(id); } - Ok(()) + Ok(digests) +} + +fn read_sparse_node( + source: &mut R, + source_node_count: usize, + node_id: MastNodeId, +) -> Result { + let tag = source.read_u8()?; + let digest = Word::read_from(source)?; + + let result = match tag { + SPARSE_BLOCK => { + let len = read_bounded_count(source, 1, "basic block data length")?; + let data = source.read_vec(len)?; + let decoder = BasicBlockDataDecoder::new(&data); + let op_batches = decoder.decode_operations(0)?; + BasicBlockNodeBuilder::from_op_batches(op_batches, digest) + .build() + .map(Into::into) + }, + SPARSE_JOIN => { + let first = read_node_id(source, source_node_count, "join first child")?; + let second = read_node_id(source, source_node_count, "join second child")?; + JoinNodeBuilder::new([first, second]) + .with_digest(digest) + .build_linked() + .map(Into::into) + }, + SPARSE_SPLIT => { + let on_true = read_node_id(source, source_node_count, "split true child")?; + let on_false = read_node_id(source, source_node_count, "split false child")?; + SplitNodeBuilder::new([on_true, on_false]) + .with_digest(digest) + .build_linked() + .map(Into::into) + }, + SPARSE_LOOP => { + let body = read_node_id(source, source_node_count, "loop body")?; + LoopNodeBuilder::new(body).with_digest(digest).build_linked().map(Into::into) + }, + SPARSE_CALL | SPARSE_SYSCALL => { + let callee = read_node_id(source, source_node_count, "call callee")?; + let builder = if tag == SPARSE_SYSCALL { + CallNodeBuilder::new_syscall(callee) + } else { + CallNodeBuilder::new(callee) + }; + builder.with_digest(digest).build_linked().map(Into::into) + }, + SPARSE_DYN | SPARSE_DYNCALL => { + let builder = if tag == SPARSE_DYNCALL { + DynNodeBuilder::new_dyncall() + } else { + DynNodeBuilder::new_dyn() + }; + Ok(builder.with_digest(digest).build().into()) + }, + SPARSE_EXTERNAL => Ok(ExternalNodeBuilder::new(digest).build().into()), + _ => { + return Err(DeserializationError::InvalidValue(format!( + "invalid sparse MAST node tag {tag}" + ))); + }, + }; + + result.map_err(|err| { + DeserializationError::InvalidValue(format!( + "failed to build sparse MAST node {}: {}", + node_id.0, err + )) + }) } -fn read_bounded_count( +fn read_bounded_count( source: &mut R, element_size: usize, label: &str, @@ -276,17 +309,12 @@ fn read_bounded_count( Ok(count) } -fn read_id_section( - source: &mut R, - count: usize, - node_count: usize, - label: &str, -) -> Result, DeserializationError> { - let mut ids = Vec::with_capacity(count); - for _ in 0..count { - ids.push(read_node_id(source, node_count, label)?); - } - Ok(ids) +fn sparse_node_min_size() -> usize { + u32::min_serialized_size() + u8::min_serialized_size() + Word::min_serialized_size() +} + +fn sparse_digest_entry_min_size() -> usize { + u32::min_serialized_size() + Word::min_serialized_size() } fn read_node_id( @@ -300,119 +328,15 @@ fn read_node_id( }) } -fn validate_strictly_increasing_ids( - ids: &[MastNodeId], - label: &str, -) -> Result<(), DeserializationError> { - for pair in ids.windows(2) { - if pair[0].0 >= pair[1].0 { - return Err(DeserializationError::InvalidValue(format!( - "{label} ids must be strictly increasing" - ))); - } - } - Ok(()) -} - -fn validate_strictly_increasing_entry_ids( - entries: &[(MastNodeId, Word)], +fn validate_strictly_increasing_id( + previous: Option, + current: MastNodeId, label: &str, ) -> Result<(), DeserializationError> { - for pair in entries.windows(2) { - if pair[0].0.0 >= pair[1].0.0 { - return Err(DeserializationError::InvalidValue(format!( - "{label} ids must be strictly increasing" - ))); - } - } - Ok(()) -} - -fn materialize_sparse_nodes( - full_ids: &[MastNodeId], - entries: &[MastNodeEntry], - full_digests: &[Word], - source_node_count: usize, - basic_block_data: &[u8], -) -> Result, DeserializationError> { - let basic_block_data_decoder = BasicBlockDataDecoder::new(basic_block_data); - if full_digests.len() != full_ids.len() { + if previous.is_some_and(|previous| previous >= current) { return Err(DeserializationError::InvalidValue(format!( - "sparse full digest count {} does not match full node count {}", - full_digests.len(), - full_ids.len() + "{label} ids must be strictly increasing" ))); } - - let mut nodes = Vec::with_capacity(entries.len()); - for ((&node_id, &entry), &digest) in full_ids.iter().zip(entries).zip(full_digests) { - let node = entry - .try_into_mast_node_builder(source_node_count, &basic_block_data_decoder, digest)? - .build_linked() - .map_err(|err| { - DeserializationError::InvalidValue(format!( - "failed to build sparse MAST node {}: {err}", - node_id.0 - )) - })?; - nodes.push((node_id, node)); - } - - Ok(nodes) -} - -impl SparseMastForest { - /// Deserializes trusted sparse MAST replay bytes using default parse budgets. - /// - /// This reader bounds parsing, but accepts sparse MAST hashes from the payload. - pub fn read_from_bytes(bytes: &[u8]) -> Result { - Self::read_from_bytes_with_options(bytes, SparseMastForestReadOptions::default()) - } - - /// Deserializes trusted sparse MAST replay bytes using explicit read options. - /// - /// See for the planned untrusted reader. - pub fn read_from_bytes_with_options( - bytes: &[u8], - options: SparseMastForestReadOptions, - ) -> Result { - let wire_byte_budget = options.wire_byte_budget(bytes.len()); - if wire_byte_budget < bytes.len() { - return Err(DeserializationError::InvalidValue( - "SparseMastForest wire byte budget is smaller than payload length".to_string(), - )); - } - let allocation_budget = wire_byte_budget.min(bytes.len().saturating_mul(4)); - let mut reader = BudgetedReader::new(SliceReader::new(bytes), allocation_budget); - let forest = read_sparse_from(&mut reader)?; - if reader.has_more_bytes() { - return Err(DeserializationError::InvalidValue( - "extra bytes after SparseMastForest payload".to_string(), - )); - } - Ok(forest) - } -} - -/// Options for reading a [`SparseMastForest`] from bytes. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct SparseMastForestReadOptions { - wire_byte_budget: Option, -} - -impl SparseMastForestReadOptions { - /// Creates options that use the default sparse read budgets. - pub fn new() -> Self { - Self::default() - } - - /// Sets the maximum number of serialized bytes consumed while parsing wire data. - pub fn with_wire_byte_budget(mut self, budget: usize) -> Self { - self.wire_byte_budget = Some(budget); - self - } - - fn wire_byte_budget(self, bytes_len: usize) -> usize { - self.wire_byte_budget.unwrap_or(bytes_len) - } + Ok(()) } diff --git a/core/src/mast/serialization/tests.rs b/core/src/mast/serialization/tests.rs index 7299cd64f5..c57085a3e5 100644 --- a/core/src/mast/serialization/tests.rs +++ b/core/src/mast/serialization/tests.rs @@ -10,10 +10,9 @@ use crate::{ chiplets::hasher, mast::{ BasicBlockNodeBuilder, CallNodeBuilder, DynNodeBuilder, ExecutableMastForest, - ExternalNodeBuilder, JoinNodeBuilder, LoopNodeBuilder, MastForestContributor, - MastForestError, MastForestView, MastNodeExt, MastNodeId, OP_BATCH_SIZE, OpBatch, - SparseMastForest, SparseMastForestBuilder, SparseMastForestReadOptions, SplitNodeBuilder, - UntrustedMastForest, UntrustedMastForestReadOptions, VisitKind, + ExternalNodeBuilder, JoinNodeBuilder, LoopNodeBuilder, MastForestError, MastForestView, + MastNodeExt, MastNodeId, OP_BATCH_SIZE, OpBatch, SparseMastForest, SparseMastForestBuilder, + SplitNodeBuilder, UntrustedMastForest, UntrustedMastForestReadOptions, VisitKind, }, operations::Operation, serde::{ByteReader, Deserializable, DeserializationError, Serializable, SliceReader}, @@ -780,7 +779,6 @@ fn sparse_mast_round_trip_preserves_sparse_replay_ids() { assert_eq!(restored.num_nodes(), source.num_nodes() as usize); assert_eq!(restored.procedure_roots(), &[root]); - assert_eq!(restored.commitment(), source.commitment()); assert_eq!( restored.get_node_by_id(true_branch).unwrap().digest(), source[true_branch].digest() @@ -816,251 +814,90 @@ fn sparse_mast_round_trip_preserves_external_full_node() { assert_eq!(restored.num_nodes(), forest.num_nodes() as usize); assert_eq!(restored.procedure_roots(), &[external]); - assert_eq!(restored.commitment(), forest.commitment()); assert_eq!(restored.get_node_by_id(external).unwrap().digest(), external_digest); assert_eq!(restored.get_digest_by_id(external), Some(external_digest)); assert!(restored.get_node_by_id(unvisited).is_none()); assert_eq!(restored.get_digest_by_id(unvisited), None); } -fn write_sparse_test_payload( - source_node_count: usize, - roots: &[MastNodeId], - full_ids: &[MastNodeId], - entries: &[MastNodeEntry], - full_digests: &[Word], - basic_block_data: &[u8], - commitment: Word, -) -> Vec { - assert_eq!(full_ids.len(), entries.len()); - assert_eq!(full_ids.len(), full_digests.len()); - - let mut bytes = Vec::new(); - bytes.write_bytes(b"SMST"); - bytes.write_bytes(&[0, 0, 0]); - - bytes.write_usize(roots.len()); - bytes.write_usize(source_node_count); - bytes.write_usize(full_ids.len()); - bytes.write_usize(0); - bytes.write_usize( - entries.iter().filter(|entry| matches!(entry, MastNodeEntry::External)).count(), - ); - bytes.write_usize(full_ids.len()); - bytes.write_usize(basic_block_data.len()); - - for root in roots { - root.0.write_into(&mut bytes); - } - commitment.write_into(&mut bytes); - bytes.write_bytes(basic_block_data); - for id in full_ids { - id.0.write_into(&mut bytes); - } - for entry in entries { - entry.write_into(&mut bytes); - } - for digest in full_digests { - digest.write_into(&mut bytes); - } - AdviceMap::default().write_into(&mut bytes); - bytes -} - -#[test] -fn sparse_reader_allows_large_source_node_count_with_small_payload() { - let block = BasicBlockNodeBuilder::new(vec![Operation::Add]).build().unwrap(); - let mut basic_block_data = BasicBlockDataBuilder::new(); - let block_offset = basic_block_data.encode_basic_block(&block); - let basic_block_data = basic_block_data.finalize(); - - let root = MastNodeId::from(0); - let bytes = write_sparse_test_payload( - MastForest::MAX_NODES, - &[root], - &[root], - &[MastNodeEntry::Block { ops_offset: block_offset }], - &[block.digest()], - &basic_block_data, - block.digest(), - ); - - let restored = SparseMastForest::read_from_bytes(&bytes).unwrap(); - assert_eq!(restored.num_nodes(), MastForest::MAX_NODES); - assert_eq!(restored.get_digest_by_id(root), Some(block.digest())); -} - #[test] -fn sparse_reader_reconstructs_forward_full_child_digests() { - let left_block = BasicBlockNodeBuilder::new(vec![Operation::Add]).build().unwrap(); - let right_block = BasicBlockNodeBuilder::new(vec![Operation::Mul]).build().unwrap(); +fn sparse_mast_round_trip_writes_map_sections_in_node_id_order() { + let mut forest = MastForest::new(); + let first = BasicBlockNodeBuilder::new(vec![Operation::Add]) + .add_to_forest(&mut forest) + .unwrap(); + let second = BasicBlockNodeBuilder::new(vec![Operation::Mul]) + .add_to_forest(&mut forest) + .unwrap(); + let third = BasicBlockNodeBuilder::new(vec![Operation::Drop]) + .add_to_forest(&mut forest) + .unwrap(); + forest.make_root(first); - let mut basic_block_data = BasicBlockDataBuilder::new(); - let left_offset = basic_block_data.encode_basic_block(&left_block); - let right_offset = basic_block_data.encode_basic_block(&right_block); - let basic_block_data = basic_block_data.finalize(); + let forest = Arc::new(forest); + let mut builder = SparseMastForestBuilder::new(Arc::clone(&forest)); + builder.record_visit(third, VisitKind::FullVisit); + builder.record_visit(first, VisitKind::FullVisit); + builder.record_visit(second, VisitKind::DigestOnly); + let sparse = builder.finalize(); - let root = MastNodeId::from(0); - let left = MastNodeId::from(1); - let right = MastNodeId::from(2); - let expected_root_digest = hasher::merge_in_domain( - &[left_block.digest(), right_block.digest()], - crate::mast::JoinNode::DOMAIN, - ); - let bytes = write_sparse_test_payload( - 3, - &[root], - &[root, left, right], - &[ - MastNodeEntry::Join { - left_child_id: left.0, - right_child_id: right.0, - }, - MastNodeEntry::Block { ops_offset: left_offset }, - MastNodeEntry::Block { ops_offset: right_offset }, - ], - &[expected_root_digest, left_block.digest(), right_block.digest()], - &basic_block_data, - expected_root_digest, - ); + let (full_ids, digest_ids) = sparse_payload_ids(&sparse.to_bytes()); - let restored = SparseMastForest::read_from_bytes(&bytes).unwrap(); - assert_eq!(restored.get_digest_by_id(root), Some(expected_root_digest)); - assert_eq!(restored.get_digest_by_id(left), Some(left_block.digest())); - assert_eq!(restored.get_digest_by_id(right), Some(right_block.digest())); + assert_eq!(full_ids, vec![first, third]); + assert_eq!(digest_ids, vec![second]); } #[test] -fn sparse_reader_preserves_forced_full_node_digest() { - let child_block = BasicBlockNodeBuilder::new(vec![Operation::Add]).build().unwrap(); - let mut basic_block_data = BasicBlockDataBuilder::new(); - let child_offset = basic_block_data.encode_basic_block(&child_block); - let basic_block_data = basic_block_data.finalize(); - - let root = MastNodeId::from(0); - let child = MastNodeId::from(1); - let canonical_root_digest = hasher::merge_in_domain( - &[child_block.digest(), Word::default()], - crate::mast::CallNode::CALL_DOMAIN, - ); - let forced_root_digest = Word::new([ - Felt::from(101_u32), - Felt::from(102_u32), - Felt::from(103_u32), - Felt::from(104_u32), - ]); - assert_ne!(forced_root_digest, canonical_root_digest); - - let bytes = write_sparse_test_payload( - 2, - &[root], - &[root, child], - &[ - MastNodeEntry::Call { callee_id: child.0 }, - MastNodeEntry::Block { ops_offset: child_offset }, - ], - &[forced_root_digest, child_block.digest()], - &basic_block_data, - forced_root_digest, - ); +fn sparse_reader_rejects_non_increasing_full_node_ids() { + let block = BasicBlockNodeBuilder::new(vec![Operation::Add]).build().unwrap(); + let mut bytes = Vec::new(); + 2usize.write_into(&mut bytes); + 0usize.write_into(&mut bytes); + 2usize.write_into(&mut bytes); + write_sparse_block_entry(MastNodeId::from(1), &block, &mut bytes); + write_sparse_block_entry(MastNodeId::from(0), &block, &mut bytes); + 0usize.write_into(&mut bytes); + AdviceMap::default().write_into(&mut bytes); - let restored = SparseMastForest::read_from_bytes(&bytes).unwrap(); - assert_eq!(restored.get_digest_by_id(root), Some(forced_root_digest)); - assert_eq!(restored.commitment(), forced_root_digest); + let err = SparseMastForest::read_from_bytes(&bytes).unwrap_err(); + assert!(err.to_string().contains("full node ids must be strictly increasing")); } #[test] -fn sparse_reader_reconstructs_deep_forward_full_child_chain() { - const CHAIN_LEN: usize = 4096; - +fn sparse_reader_rejects_non_increasing_digest_only_ids() { let block = BasicBlockNodeBuilder::new(vec![Operation::Add]).build().unwrap(); - let mut basic_block_data = BasicBlockDataBuilder::new(); - let block_offset = basic_block_data.encode_basic_block(&block); - let basic_block_data = basic_block_data.finalize(); - - let full_ids: Vec<_> = (0..CHAIN_LEN).map(|id| MastNodeId::from(id as u32)).collect(); - let mut entries = Vec::with_capacity(CHAIN_LEN); - for id in 0..CHAIN_LEN - 1 { - entries.push(MastNodeEntry::Call { callee_id: (id + 1) as u32 }); - } - entries.push(MastNodeEntry::Block { ops_offset: block_offset }); - - let mut full_digests = vec![Word::default(); CHAIN_LEN]; - full_digests[CHAIN_LEN - 1] = block.digest(); - for id in (0..CHAIN_LEN - 1).rev() { - full_digests[id] = hasher::merge_in_domain( - &[full_digests[id + 1], Word::default()], - crate::mast::CallNode::CALL_DOMAIN, - ); - } - let expected_root_digest = full_digests[0]; - - let root = MastNodeId::from(0); - let bytes = write_sparse_test_payload( - CHAIN_LEN, - &[root], - &full_ids, - &entries, - &full_digests, - &basic_block_data, - expected_root_digest, - ); + let mut bytes = Vec::new(); + 3usize.write_into(&mut bytes); + 0usize.write_into(&mut bytes); + 1usize.write_into(&mut bytes); + write_sparse_block_entry(MastNodeId::from(0), &block, &mut bytes); + 2usize.write_into(&mut bytes); + MastNodeId::from(2).write_into(&mut bytes); + block.digest().write_into(&mut bytes); + MastNodeId::from(1).write_into(&mut bytes); + block.digest().write_into(&mut bytes); + AdviceMap::default().write_into(&mut bytes); - let restored = SparseMastForest::read_from_bytes(&bytes).unwrap(); - assert_eq!(restored.get_digest_by_id(root), Some(expected_root_digest)); + let err = SparseMastForest::read_from_bytes(&bytes).unwrap_err(); + assert!(err.to_string().contains("digest-only node ids must be strictly increasing")); } #[test] -fn sparse_reader_rejects_trailing_bytes_with_exact_prefix_budget() { +fn sparse_reader_rejects_trailing_bytes() { let block = BasicBlockNodeBuilder::new(vec![Operation::Add]).build().unwrap(); - let mut basic_block_data = BasicBlockDataBuilder::new(); - let block_offset = basic_block_data.encode_basic_block(&block); - let basic_block_data = basic_block_data.finalize(); - - let root = MastNodeId::from(0); - let mut bytes_with_trailing = write_sparse_test_payload( - 1, - &[root], - &[root], - &[MastNodeEntry::Block { ops_offset: block_offset }], - &[block.digest()], - &basic_block_data, - block.digest(), - ); + let mut bytes = Vec::new(); + 1usize.write_into(&mut bytes); + 0usize.write_into(&mut bytes); + 1usize.write_into(&mut bytes); + write_sparse_block_entry(MastNodeId::from(0), &block, &mut bytes); + 0usize.write_into(&mut bytes); + AdviceMap::default().write_into(&mut bytes); + bytes.push(0); - bytes_with_trailing.push(0); - let err = SparseMastForest::read_from_bytes_with_options( - &bytes_with_trailing, - SparseMastForestReadOptions::new().with_wire_byte_budget(bytes_with_trailing.len()), - ) - .unwrap_err(); + let err = SparseMastForest::read_from_bytes(&bytes).unwrap_err(); assert!(err.to_string().contains("extra bytes after SparseMastForest payload")); } -#[test] -fn dense_mast_readers_reject_sparse_payloads() { - let (_source, sparse, _true_branch, _false_branch, _root) = sparse_split_fixture(); - let bytes = sparse.to_bytes(); - - let materialized = MastForest::read_from_bytes(&bytes); - assert_matches!( - materialized, - Err(DeserializationError::InvalidValue(msg)) if msg.contains("Invalid magic bytes") - ); - - let wire_view = MastForestWireView::new(&bytes); - assert_matches!( - wire_view, - Err(DeserializationError::InvalidValue(msg)) if msg.contains("Invalid magic bytes") - ); - - let untrusted = UntrustedMastForest::read_from_bytes(&bytes); - assert_matches!( - untrusted, - Err(DeserializationError::InvalidValue(msg)) if msg.contains("Invalid magic bytes") - ); -} - #[test] fn sparse_reader_rejects_dense_payloads() { let mut forest = MastForest::new(); @@ -1070,10 +907,7 @@ fn sparse_reader_rejects_dense_payloads() { forest.make_root(root); let result = SparseMastForest::read_from_bytes(&forest.to_bytes()); - assert_matches!( - result, - Err(DeserializationError::InvalidValue(msg)) if msg.contains("Invalid sparse MAST magic bytes") - ); + assert!(result.is_err()); } #[test] @@ -1092,7 +926,6 @@ fn sparse_serialized_parts_reject_missing_child_digest() { sparse.num_nodes(), sparse.procedure_roots().to_vec(), sparse.advice_map().clone(), - sparse.commitment(), ); assert_matches!( @@ -1115,7 +948,6 @@ fn sparse_serialized_parts_reject_duplicate_full_ids() { sparse.num_nodes(), sparse.procedure_roots().to_vec(), sparse.advice_map().clone(), - sparse.commitment(), ); assert_matches!( @@ -1139,7 +971,6 @@ fn sparse_serialized_parts_reject_duplicate_digest_only_ids() { sparse.num_nodes(), sparse.procedure_roots().to_vec(), sparse.advice_map().clone(), - sparse.commitment(), ); assert_matches!( @@ -1162,7 +993,6 @@ fn sparse_serialized_parts_reject_full_digest_overlap() { sparse.num_nodes(), sparse.procedure_roots().to_vec(), sparse.advice_map().clone(), - sparse.commitment(), ); assert_matches!( @@ -1186,7 +1016,6 @@ fn sparse_serialized_parts_reject_out_of_range_full_digest_and_root_ids() { sparse.num_nodes(), sparse.procedure_roots().to_vec(), sparse.advice_map().clone(), - sparse.commitment(), ); assert_matches!( result, @@ -1203,7 +1032,6 @@ fn sparse_serialized_parts_reject_out_of_range_full_digest_and_root_ids() { sparse.num_nodes(), sparse.procedure_roots().to_vec(), sparse.advice_map().clone(), - sparse.commitment(), ); assert_matches!( result, @@ -1219,7 +1047,6 @@ fn sparse_serialized_parts_reject_out_of_range_full_digest_and_root_ids() { sparse.num_nodes(), vec![out_of_range], sparse.advice_map().clone(), - sparse.commitment(), ); assert_matches!( result, @@ -1228,6 +1055,71 @@ fn sparse_serialized_parts_reject_out_of_range_full_digest_and_root_ids() { ); } +fn write_sparse_block_entry( + id: MastNodeId, + block: &crate::mast::BasicBlockNode, + target: &mut W, +) { + id.write_into(target); + target.write_u8(0); + block.digest().write_into(target); + + let mut basic_block_data = BasicBlockDataBuilder::new(); + let ops_offset = basic_block_data.encode_basic_block(block); + assert_eq!(ops_offset, 0); + let basic_block_data = basic_block_data.finalize(); + target.write_usize(basic_block_data.len()); + target.write_bytes(&basic_block_data); +} + +fn sparse_payload_ids(bytes: &[u8]) -> (Vec, Vec) { + let mut reader = SliceReader::new(bytes); + let _num_nodes = reader.read_usize().unwrap(); + + let root_count = reader.read_usize().unwrap(); + for _ in 0..root_count { + let _root = MastNodeId::read_from(&mut reader).unwrap(); + } + + let full_count = reader.read_usize().unwrap(); + let mut full_ids = Vec::new(); + for _ in 0..full_count { + let id = MastNodeId::read_from(&mut reader).unwrap(); + skip_sparse_node(&mut reader); + full_ids.push(id); + } + + let digest_count = reader.read_usize().unwrap(); + let mut digest_ids = Vec::new(); + for _ in 0..digest_count { + let id = MastNodeId::read_from(&mut reader).unwrap(); + let _digest = Word::read_from(&mut reader).unwrap(); + digest_ids.push(id); + } + + (full_ids, digest_ids) +} + +fn skip_sparse_node(reader: &mut SliceReader<'_>) { + let tag = reader.read_u8().unwrap(); + let _digest = Word::read_from(reader).unwrap(); + match tag { + 0 => { + let len = reader.read_usize().unwrap(); + let _ = reader.read_slice(len).unwrap(); + }, + 1 | 2 => { + let _first = MastNodeId::read_from(reader).unwrap(); + let _second = MastNodeId::read_from(reader).unwrap(); + }, + 3..=5 => { + let _child = MastNodeId::read_from(reader).unwrap(); + }, + 6..=8 => {}, + _ => panic!("unexpected sparse test node tag {tag}"), + } +} + /// Test that a forest with a node whose child ids are larger than its own id serializes and /// deserializes successfully. #[test] diff --git a/core/src/mast/sparse.rs b/core/src/mast/sparse.rs index 6ff7d6cc05..f6fb7aa443 100644 --- a/core/src/mast/sparse.rs +++ b/core/src/mast/sparse.rs @@ -63,9 +63,6 @@ pub struct SparseMastForest { /// Advice map to be loaded into the VM prior to executing procedures from this MAST forest. advice_map: AdviceMap, - - /// Cached commitment to the original MAST forest (i.e. a commitment to all roots). - commitment_cache: Word, } impl SparseMastForest { @@ -97,23 +94,13 @@ impl SparseMastForest { &self.digests } - /// Returns the commitment to this sparse forest, computed from the procedure roots. - /// - /// The commitment value is derived from the digests of the procedure roots in the original - /// forest; it is therefore equal to the commitment of the source [`MastForest`] from which - /// this sparse forest was built. - pub fn commitment(&self) -> Word { - self.commitment_cache - } - - /// Builds a sparse forest from parts decoded from the sparse wire format. + /// Builds a sparse forest from trusted replay parts. pub(in crate::mast) fn from_serialized_parts( nodes: Vec<(MastNodeId, MastNode)>, digests: Vec<(MastNodeId, Word)>, num_nodes: usize, roots: Vec, advice_map: AdviceMap, - commitment_cache: Word, ) -> Result { validate_sparse_node_bound(num_nodes)?; @@ -141,7 +128,6 @@ impl SparseMastForest { num_nodes, roots, advice_map, - commitment_cache, }) } } @@ -404,7 +390,6 @@ impl SparseMastForestBuilder { num_nodes, roots: source.procedure_roots().to_vec(), advice_map: source.advice_map().clone(), - commitment_cache: source.commitment(), } } } From 3d07e498461bfbb82b1b5b1a133f58ba08ab9abf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Garillot?= Date: Fri, 10 Jul 2026 13:17:21 -0400 Subject: [PATCH 6/9] fix: bound sparse MAST replay reads --- core/src/mast/serialization/sparse.rs | 31 ++++++++++++++++----------- core/src/mast/serialization/tests.rs | 22 +++++++++++++++++++ 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/core/src/mast/serialization/sparse.rs b/core/src/mast/serialization/sparse.rs index a4277153cd..f3eeed9270 100644 --- a/core/src/mast/serialization/sparse.rs +++ b/core/src/mast/serialization/sparse.rs @@ -1,16 +1,20 @@ use alloc::{collections::BTreeMap, format, string::ToString, vec::Vec}; -use super::basic_blocks::{BasicBlockDataBuilder, BasicBlockDataDecoder}; +use super::{ + TRUSTED_BYTE_READ_BUDGET_MULTIPLIER, + basic_blocks::{BasicBlockDataBuilder, BasicBlockDataDecoder}, +}; use crate::{ Word, advice::AdviceMap, mast::{ BasicBlockNodeBuilder, CallNodeBuilder, DynNodeBuilder, ExternalNodeBuilder, - JoinNodeBuilder, LoopNodeBuilder, MastForestContributor, MastNode, MastNodeExt, MastNodeId, - SparseMastForest, SplitNodeBuilder, + JoinNodeBuilder, LoopNodeBuilder, MastForest, MastForestContributor, MastNode, MastNodeExt, + MastNodeId, SparseMastForest, SplitNodeBuilder, }, serde::{ - ByteReader, ByteWriter, Deserializable, DeserializationError, Serializable, SliceReader, + BudgetedReader, ByteReader, ByteWriter, Deserializable, DeserializationError, Serializable, + SliceReader, read_bounded_len, }, }; @@ -139,7 +143,8 @@ impl Deserializable for SparseMastForest { /// bytes from a trusted producer, or after an outer transport/authentication layer has accepted /// them. fn read_from_bytes(bytes: &[u8]) -> Result { - let mut reader = SliceReader::new(bytes); + let budget = bytes.len().saturating_mul(TRUSTED_BYTE_READ_BUDGET_MULTIPLIER); + let mut reader = BudgetedReader::new(SliceReader::new(bytes), budget); let forest = read_sparse_from(&mut reader)?; if reader.has_more_bytes() { return Err(DeserializationError::InvalidValue( @@ -157,6 +162,13 @@ fn read_sparse_from( source: &mut R, ) -> Result { let source_node_count = source.read_usize()?; + if source_node_count > MastForest::MAX_NODES { + return Err(DeserializationError::InvalidValue(format!( + "sparse source node count {source_node_count} exceeds maximum allowed {}", + MastForest::MAX_NODES + ))); + } + let roots = read_node_ids(source, source_node_count, "procedure root")?; let nodes = read_sparse_nodes(source, source_node_count)?; let digest_entries = read_digest_entries(source, source_node_count)?; @@ -299,14 +311,7 @@ fn read_bounded_count( element_size: usize, label: &str, ) -> Result { - let count = source.read_usize()?; - let max_count = source.max_alloc(element_size); - if count > max_count { - return Err(DeserializationError::InvalidValue(format!( - "{label} {count} exceeds reader allocation bound {max_count} for {element_size}-byte elements" - ))); - } - Ok(count) + read_bounded_len(source, label, element_size) } fn sparse_node_min_size() -> usize { diff --git a/core/src/mast/serialization/tests.rs b/core/src/mast/serialization/tests.rs index c57085a3e5..7b080b7197 100644 --- a/core/src/mast/serialization/tests.rs +++ b/core/src/mast/serialization/tests.rs @@ -910,6 +910,28 @@ fn sparse_reader_rejects_dense_payloads() { assert!(result.is_err()); } +#[test] +fn sparse_reader_rejects_oversized_source_node_count_before_sections() { + let mut bytes = Vec::new(); + (MastForest::MAX_NODES + 1).write_into(&mut bytes); + + let err = SparseMastForest::read_from_bytes(&bytes).unwrap_err(); + + assert!(err.to_string().contains("sparse source node count")); + assert!(err.to_string().contains("exceeds maximum allowed")); +} + +#[test] +fn sparse_reader_rejects_oversized_section_count_before_allocation() { + let mut bytes = Vec::new(); + 1usize.write_into(&mut bytes); + usize::MAX.write_into(&mut bytes); + + let err = SparseMastForest::read_from_bytes(&bytes).unwrap_err(); + + assert!(err.to_string().contains("procedure root count")); +} + #[test] fn sparse_serialized_parts_reject_missing_child_digest() { let (_source, sparse, _true_branch, false_branch, _root) = sparse_split_fixture(); From 213b2051ee1be9a0a60fc38fb002033992dca35d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Garillot?= Date: Fri, 10 Jul 2026 15:28:49 -0400 Subject: [PATCH 7/9] refactor: remove sparse MAST source count --- core/src/mast/serialization/sparse.rs | 50 ++++--------- core/src/mast/serialization/tests.rs | 29 +++----- core/src/mast/sparse.rs | 101 ++++++++------------------ 3 files changed, 56 insertions(+), 124 deletions(-) diff --git a/core/src/mast/serialization/sparse.rs b/core/src/mast/serialization/sparse.rs index f3eeed9270..9408466517 100644 --- a/core/src/mast/serialization/sparse.rs +++ b/core/src/mast/serialization/sparse.rs @@ -38,7 +38,6 @@ const SPARSE_EXTERNAL: u8 = 8; /// [`MastForest`] wire format. Callers must only read these bytes from a trusted producer, or after /// an outer transport/authentication layer has accepted them. fn write_sparse_into(forest: &SparseMastForest, target: &mut W) { - forest.num_nodes().write_into(target); write_node_ids(forest.procedure_roots(), target); write_sparse_nodes(forest.nodes(), target); write_digest_entries(forest.digest_entries(), target); @@ -161,53 +160,37 @@ impl Deserializable for SparseMastForest { fn read_sparse_from( source: &mut R, ) -> Result { - let source_node_count = source.read_usize()?; - if source_node_count > MastForest::MAX_NODES { - return Err(DeserializationError::InvalidValue(format!( - "sparse source node count {source_node_count} exceeds maximum allowed {}", - MastForest::MAX_NODES - ))); - } - - let roots = read_node_ids(source, source_node_count, "procedure root")?; - let nodes = read_sparse_nodes(source, source_node_count)?; - let digest_entries = read_digest_entries(source, source_node_count)?; + let roots = read_node_ids(source, "procedure root")?; + let nodes = read_sparse_nodes(source)?; + let digest_entries = read_digest_entries(source)?; let advice_map = AdviceMap::read_from(source)?; - SparseMastForest::from_serialized_parts( - nodes, - digest_entries, - source_node_count, - roots, - advice_map, - ) + SparseMastForest::from_serialized_parts(nodes, digest_entries, roots, advice_map) } fn read_node_ids( source: &mut R, - source_node_count: usize, label: &str, ) -> Result, DeserializationError> { let count = read_bounded_count(source, u32::min_serialized_size(), label)?; let mut ids = Vec::with_capacity(count); for _ in 0..count { - ids.push(read_node_id(source, source_node_count, label)?); + ids.push(read_node_id(source, label)?); } Ok(ids) } fn read_sparse_nodes( source: &mut R, - source_node_count: usize, ) -> Result, DeserializationError> { let count = read_bounded_count(source, sparse_node_min_size(), "full node count")?; let mut nodes = Vec::with_capacity(count); let mut previous_id = None; for _ in 0..count { - let id = read_node_id(source, source_node_count, "full node")?; + let id = read_node_id(source, "full node")?; validate_strictly_increasing_id(previous_id, id, "full node")?; - let node = read_sparse_node(source, source_node_count, id)?; + let node = read_sparse_node(source, id)?; nodes.push((id, node)); previous_id = Some(id); } @@ -217,7 +200,6 @@ fn read_sparse_nodes( fn read_digest_entries( source: &mut R, - source_node_count: usize, ) -> Result, DeserializationError> { let count = read_bounded_count(source, sparse_digest_entry_min_size(), "digest-only node count")?; @@ -225,7 +207,7 @@ fn read_digest_entries( let mut previous_id = None; for _ in 0..count { - let id = read_node_id(source, source_node_count, "digest-only node")?; + let id = read_node_id(source, "digest-only node")?; validate_strictly_increasing_id(previous_id, id, "digest-only node")?; let digest = Word::read_from(source)?; digests.push((id, digest)); @@ -237,7 +219,6 @@ fn read_digest_entries( fn read_sparse_node( source: &mut R, - source_node_count: usize, node_id: MastNodeId, ) -> Result { let tag = source.read_u8()?; @@ -254,27 +235,27 @@ fn read_sparse_node( .map(Into::into) }, SPARSE_JOIN => { - let first = read_node_id(source, source_node_count, "join first child")?; - let second = read_node_id(source, source_node_count, "join second child")?; + let first = read_node_id(source, "join first child")?; + let second = read_node_id(source, "join second child")?; JoinNodeBuilder::new([first, second]) .with_digest(digest) .build_linked() .map(Into::into) }, SPARSE_SPLIT => { - let on_true = read_node_id(source, source_node_count, "split true child")?; - let on_false = read_node_id(source, source_node_count, "split false child")?; + let on_true = read_node_id(source, "split true child")?; + let on_false = read_node_id(source, "split false child")?; SplitNodeBuilder::new([on_true, on_false]) .with_digest(digest) .build_linked() .map(Into::into) }, SPARSE_LOOP => { - let body = read_node_id(source, source_node_count, "loop body")?; + let body = read_node_id(source, "loop body")?; LoopNodeBuilder::new(body).with_digest(digest).build_linked().map(Into::into) }, SPARSE_CALL | SPARSE_SYSCALL => { - let callee = read_node_id(source, source_node_count, "call callee")?; + let callee = read_node_id(source, "call callee")?; let builder = if tag == SPARSE_SYSCALL { CallNodeBuilder::new_syscall(callee) } else { @@ -324,11 +305,10 @@ fn sparse_digest_entry_min_size() -> usize { fn read_node_id( source: &mut R, - node_count: usize, label: &str, ) -> Result { let raw = u32::read_from(source)?; - MastNodeId::from_u32_with_node_count(raw, node_count).map_err(|err| { + MastNodeId::from_u32_with_node_count(raw, MastForest::MAX_NODES).map_err(|err| { DeserializationError::InvalidValue(format!("invalid {label} id {raw}: {err}")) }) } diff --git a/core/src/mast/serialization/tests.rs b/core/src/mast/serialization/tests.rs index 7b080b7197..08adafb8da 100644 --- a/core/src/mast/serialization/tests.rs +++ b/core/src/mast/serialization/tests.rs @@ -851,7 +851,6 @@ fn sparse_mast_round_trip_writes_map_sections_in_node_id_order() { fn sparse_reader_rejects_non_increasing_full_node_ids() { let block = BasicBlockNodeBuilder::new(vec![Operation::Add]).build().unwrap(); let mut bytes = Vec::new(); - 2usize.write_into(&mut bytes); 0usize.write_into(&mut bytes); 2usize.write_into(&mut bytes); write_sparse_block_entry(MastNodeId::from(1), &block, &mut bytes); @@ -867,7 +866,6 @@ fn sparse_reader_rejects_non_increasing_full_node_ids() { fn sparse_reader_rejects_non_increasing_digest_only_ids() { let block = BasicBlockNodeBuilder::new(vec![Operation::Add]).build().unwrap(); let mut bytes = Vec::new(); - 3usize.write_into(&mut bytes); 0usize.write_into(&mut bytes); 1usize.write_into(&mut bytes); write_sparse_block_entry(MastNodeId::from(0), &block, &mut bytes); @@ -886,7 +884,6 @@ fn sparse_reader_rejects_non_increasing_digest_only_ids() { fn sparse_reader_rejects_trailing_bytes() { let block = BasicBlockNodeBuilder::new(vec![Operation::Add]).build().unwrap(); let mut bytes = Vec::new(); - 1usize.write_into(&mut bytes); 0usize.write_into(&mut bytes); 1usize.write_into(&mut bytes); write_sparse_block_entry(MastNodeId::from(0), &block, &mut bytes); @@ -911,20 +908,20 @@ fn sparse_reader_rejects_dense_payloads() { } #[test] -fn sparse_reader_rejects_oversized_source_node_count_before_sections() { +fn sparse_reader_rejects_oversized_node_id_before_sections() { let mut bytes = Vec::new(); - (MastForest::MAX_NODES + 1).write_into(&mut bytes); + 1usize.write_into(&mut bytes); + MastNodeId::from(MastForest::MAX_NODES as u32).write_into(&mut bytes); let err = SparseMastForest::read_from_bytes(&bytes).unwrap_err(); - assert!(err.to_string().contains("sparse source node count")); - assert!(err.to_string().contains("exceeds maximum allowed")); + assert!(err.to_string().contains("procedure root id")); + assert!(err.to_string().contains("number of nodes in the forest")); } #[test] fn sparse_reader_rejects_oversized_section_count_before_allocation() { let mut bytes = Vec::new(); - 1usize.write_into(&mut bytes); usize::MAX.write_into(&mut bytes); let err = SparseMastForest::read_from_bytes(&bytes).unwrap_err(); @@ -945,7 +942,6 @@ fn sparse_serialized_parts_reject_missing_child_digest() { let result = SparseMastForest::from_serialized_parts( nodes, digests, - sparse.num_nodes(), sparse.procedure_roots().to_vec(), sparse.advice_map().clone(), ); @@ -967,7 +963,6 @@ fn sparse_serialized_parts_reject_duplicate_full_ids() { let result = SparseMastForest::from_serialized_parts( nodes, digests, - sparse.num_nodes(), sparse.procedure_roots().to_vec(), sparse.advice_map().clone(), ); @@ -990,7 +985,6 @@ fn sparse_serialized_parts_reject_duplicate_digest_only_ids() { let result = SparseMastForest::from_serialized_parts( nodes, digests, - sparse.num_nodes(), sparse.procedure_roots().to_vec(), sparse.advice_map().clone(), ); @@ -1012,7 +1006,6 @@ fn sparse_serialized_parts_reject_full_digest_overlap() { let result = SparseMastForest::from_serialized_parts( nodes, digests, - sparse.num_nodes(), sparse.procedure_roots().to_vec(), sparse.advice_map().clone(), ); @@ -1026,7 +1019,7 @@ fn sparse_serialized_parts_reject_full_digest_overlap() { #[test] fn sparse_serialized_parts_reject_out_of_range_full_digest_and_root_ids() { let (_source, sparse, true_branch, false_branch, _root) = sparse_split_fixture(); - let out_of_range = MastNodeId::from(sparse.num_nodes() as u32); + let out_of_range = MastNodeId::from(MastForest::MAX_NODES as u32); let mut nodes: Vec<_> = sparse.nodes().iter().map(|(&id, node)| (id, node.clone())).collect(); nodes.push((out_of_range, sparse.get_node_by_id(true_branch).unwrap().clone())); @@ -1035,14 +1028,13 @@ fn sparse_serialized_parts_reject_out_of_range_full_digest_and_root_ids() { let result = SparseMastForest::from_serialized_parts( nodes, digests.clone(), - sparse.num_nodes(), sparse.procedure_roots().to_vec(), sparse.advice_map().clone(), ); assert_matches!( result, Err(DeserializationError::InvalidValue(msg)) if msg.contains("full node id") - && msg.contains("out of range") + && msg.contains("exceeds maximum") ); let nodes = sparse.nodes().iter().map(|(&id, node)| (id, node.clone())).collect(); @@ -1051,14 +1043,13 @@ fn sparse_serialized_parts_reject_out_of_range_full_digest_and_root_ids() { let result = SparseMastForest::from_serialized_parts( nodes, out_of_range_digests, - sparse.num_nodes(), sparse.procedure_roots().to_vec(), sparse.advice_map().clone(), ); assert_matches!( result, Err(DeserializationError::InvalidValue(msg)) if msg.contains("digest-only node id") - && msg.contains("out of range") + && msg.contains("exceeds maximum") ); let nodes = sparse.nodes().iter().map(|(&id, node)| (id, node.clone())).collect(); @@ -1066,14 +1057,13 @@ fn sparse_serialized_parts_reject_out_of_range_full_digest_and_root_ids() { let result = SparseMastForest::from_serialized_parts( nodes, digests, - sparse.num_nodes(), vec![out_of_range], sparse.advice_map().clone(), ); assert_matches!( result, Err(DeserializationError::InvalidValue(msg)) if msg.contains("procedure root id") - && msg.contains("out of range") + && msg.contains("exceeds maximum") ); } @@ -1096,7 +1086,6 @@ fn write_sparse_block_entry( fn sparse_payload_ids(bytes: &[u8]) -> (Vec, Vec) { let mut reader = SliceReader::new(bytes); - let _num_nodes = reader.read_usize().unwrap(); let root_count = reader.read_usize().unwrap(); for _ in 0..root_count { diff --git a/core/src/mast/sparse.rs b/core/src/mast/sparse.rs index f6fb7aa443..f2b9f69057 100644 --- a/core/src/mast/sparse.rs +++ b/core/src/mast/sparse.rs @@ -52,12 +52,6 @@ pub struct SparseMastForest { /// full-node entry implicitly carries its own digest via [`MastNodeExt::digest`]. digests: BTreeMap, - /// Upper bound for [`MastNodeId`] values from the source [`MastForest`]. - /// - /// This is not `nodes.len()`. The trusted reader uses it to reject IDs outside the source - /// forest's ID space, not as an independent proof of the source forest's size. - num_nodes: usize, - /// Roots of procedures defined within the original MAST forest. roots: Vec, @@ -72,11 +66,18 @@ impl SparseMastForest { &self.nodes } - /// Returns the total number of nodes in the source [`MastForest`] from which this sparse - /// forest was built. This is *not* the number of visited (i.e. present) nodes — see - /// [`Self::nodes`] for that. + /// Returns the minimum node count needed to cover all IDs retained in this sparse replay view. + /// + /// This is *not* the number of visited nodes and may be smaller than the source + /// [`MastForest`]'s node count when high source IDs were not needed during replay. pub fn num_nodes(&self) -> usize { - self.num_nodes + self.nodes + .keys() + .chain(self.digests.keys()) + .chain(self.roots.iter()) + .map(|id| id.to_usize() + 1) + .max() + .unwrap_or(0) } /// Returns the roots of procedures defined within this sparse forest. @@ -98,17 +99,14 @@ impl SparseMastForest { pub(in crate::mast) fn from_serialized_parts( nodes: Vec<(MastNodeId, MastNode)>, digests: Vec<(MastNodeId, Word)>, - num_nodes: usize, roots: Vec, advice_map: AdviceMap, ) -> Result { - validate_sparse_node_bound(num_nodes)?; - - let nodes = collect_unique_nodes(nodes, num_nodes)?; - let digests = collect_unique_digests(digests, num_nodes)?; + let nodes = collect_unique_nodes(nodes)?; + let digests = collect_unique_digests(digests)?; for &root in &roots { - validate_sparse_id(root, num_nodes, "procedure root")?; + validate_sparse_id(root, "procedure root")?; } for node_id in nodes.keys() { @@ -120,37 +118,18 @@ impl SparseMastForest { } } - validate_full_node_child_digests(&nodes, &digests, num_nodes)?; - - Ok(Self { - nodes, - digests, - num_nodes, - roots, - advice_map, - }) - } -} + validate_full_node_child_digests(&nodes, &digests)?; -fn validate_sparse_node_bound(num_nodes: usize) -> Result<(), DeserializationError> { - if num_nodes > MastForest::MAX_NODES { - return Err(DeserializationError::InvalidValue(format!( - "sparse source node count {num_nodes} exceeds maximum allowed {}", - MastForest::MAX_NODES - ))); + Ok(Self { nodes, digests, roots, advice_map }) } - Ok(()) } -fn validate_sparse_id( - id: MastNodeId, - num_nodes: usize, - label: &str, -) -> Result<(), DeserializationError> { - if id.to_usize() >= num_nodes { +fn validate_sparse_id(id: MastNodeId, label: &str) -> Result<(), DeserializationError> { + if id.to_usize() >= MastForest::MAX_NODES { return Err(DeserializationError::InvalidValue(format!( - "{label} id {} is out of range for sparse source node count {num_nodes}", - id.0 + "{label} id {} exceeds maximum sparse MAST node id {}", + id.0, + MastForest::MAX_NODES - 1 ))); } Ok(()) @@ -158,11 +137,10 @@ fn validate_sparse_id( fn collect_unique_nodes( nodes: Vec<(MastNodeId, MastNode)>, - num_nodes: usize, ) -> Result, DeserializationError> { let mut result = BTreeMap::new(); for (id, node) in nodes { - validate_sparse_id(id, num_nodes, "full node")?; + validate_sparse_id(id, "full node")?; if result.insert(id, node).is_some() { return Err(DeserializationError::InvalidValue(format!( "duplicate sparse full-node id {}", @@ -175,11 +153,10 @@ fn collect_unique_nodes( fn collect_unique_digests( digests: Vec<(MastNodeId, Word)>, - num_nodes: usize, ) -> Result, DeserializationError> { let mut result = BTreeMap::new(); for (id, digest) in digests { - validate_sparse_id(id, num_nodes, "digest-only node")?; + validate_sparse_id(id, "digest-only node")?; if result.insert(id, digest).is_some() { return Err(DeserializationError::InvalidValue(format!( "duplicate sparse digest-only id {}", @@ -195,10 +172,9 @@ fn collect_unique_digests( fn validate_full_node_child_digests( nodes: &BTreeMap, digests: &BTreeMap, - num_nodes: usize, ) -> Result<(), DeserializationError> { for (&node_id, node) in nodes { - validate_sparse_id(node_id, num_nodes, "full node")?; + validate_sparse_id(node_id, "full node")?; match node { MastNode::Block(block) => { @@ -211,18 +187,18 @@ fn validate_full_node_child_digests( }, MastNode::External(_) | MastNode::Dyn(_) => {}, MastNode::Join(join) => { - require_child_digest(node_id, join.first(), nodes, digests, num_nodes)?; - require_child_digest(node_id, join.second(), nodes, digests, num_nodes)?; + require_child_digest(node_id, join.first(), nodes, digests)?; + require_child_digest(node_id, join.second(), nodes, digests)?; }, MastNode::Split(split) => { - require_child_digest(node_id, split.on_true(), nodes, digests, num_nodes)?; - require_child_digest(node_id, split.on_false(), nodes, digests, num_nodes)?; + require_child_digest(node_id, split.on_true(), nodes, digests)?; + require_child_digest(node_id, split.on_false(), nodes, digests)?; }, MastNode::Loop(loop_node) => { - require_child_digest(node_id, loop_node.body(), nodes, digests, num_nodes)?; + require_child_digest(node_id, loop_node.body(), nodes, digests)?; }, MastNode::Call(call) => { - require_child_digest(node_id, call.callee(), nodes, digests, num_nodes)?; + require_child_digest(node_id, call.callee(), nodes, digests)?; }, } } @@ -234,9 +210,8 @@ fn require_child_digest( child_id: MastNodeId, nodes: &BTreeMap, digests: &BTreeMap, - num_nodes: usize, ) -> Result<(), DeserializationError> { - validate_sparse_id(child_id, num_nodes, "child")?; + validate_sparse_id(child_id, "child")?; if !nodes.contains_key(&child_id) && !digests.contains_key(&child_id) { return Err(DeserializationError::InvalidValue(format!( "sparse full node {} references child {} without a full node or digest-only entry", @@ -308,10 +283,6 @@ pub struct SparseMastForestBuilder { /// The source forest whose nodes are being collected. source: Arc, - /// Total number of nodes in the source forest, captured at construction time. Propagated to - /// the finalized [`SparseMastForest`] so consumers know the original [`MastNodeId`] space. - num_nodes: usize, - /// IDs of nodes that were entered during execution. Their full [`MastNode`] is copied into the /// finalized forest's `nodes` map. full_visits: BTreeSet, @@ -325,10 +296,8 @@ pub struct SparseMastForestBuilder { impl SparseMastForestBuilder { /// Creates a new builder for the given source forest. pub fn new(source: Arc) -> Self { - let num_nodes = source.nodes().len(); Self { source, - num_nodes, full_visits: BTreeSet::new(), digest_only_visits: BTreeSet::new(), } @@ -358,12 +327,7 @@ impl SparseMastForestBuilder { /// from the source forest. The roots, advice map, and debug info are cloned from the source /// in full (they are not yet trimmed to visited nodes only). pub fn finalize(self) -> SparseMastForest { - let SparseMastForestBuilder { - source, - num_nodes, - full_visits, - digest_only_visits, - } = self; + let SparseMastForestBuilder { source, full_visits, digest_only_visits } = self; let mut nodes = BTreeMap::new(); for node_id in &full_visits { @@ -387,7 +351,6 @@ impl SparseMastForestBuilder { SparseMastForest { nodes, digests, - num_nodes, roots: source.procedure_roots().to_vec(), advice_map: source.advice_map().clone(), } From 15c371736e5ed8e2c408d7cc6604415aca6b55c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Garillot?= Date: Fri, 10 Jul 2026 22:12:46 -0400 Subject: [PATCH 8/9] fix: keep sparse MAST advice empty --- core/src/mast/serialization/tests.rs | 45 ++++++++++++++++++++++++++++ core/src/mast/sparse.rs | 25 ++++++++++++---- 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/core/src/mast/serialization/tests.rs b/core/src/mast/serialization/tests.rs index 08adafb8da..9ddb6ac088 100644 --- a/core/src/mast/serialization/tests.rs +++ b/core/src/mast/serialization/tests.rs @@ -847,6 +847,51 @@ fn sparse_mast_round_trip_writes_map_sections_in_node_id_order() { assert_eq!(digest_ids, vec![second]); } +#[test] +fn sparse_mast_drops_source_advice_map() { + let mut source = MastForest::new(); + let root = BasicBlockNodeBuilder::new(vec![Operation::Add]) + .add_to_forest(&mut source) + .unwrap(); + source.make_root(root); + let advice_key = Word::new([ + Felt::new_unchecked(11), + Felt::new_unchecked(12), + Felt::new_unchecked(13), + Felt::new_unchecked(14), + ]); + let advice_values = vec![Felt::new_unchecked(15), Felt::new_unchecked(16)]; + let source = source.with_advice_map(AdviceMap::from_iter([(advice_key, advice_values)])); + + let mut builder = SparseMastForestBuilder::new(Arc::new(source)); + builder.record_visit(root, VisitKind::FullVisit); + let sparse = builder.finalize(); + let restored = SparseMastForest::read_from_bytes(&sparse.to_bytes()).unwrap(); + + assert!(sparse.advice_map().is_empty()); + assert!(restored.advice_map().is_empty()); +} + +#[test] +fn sparse_reader_rejects_non_empty_advice_map() { + let advice_key = Word::new([ + Felt::new_unchecked(21), + Felt::new_unchecked(22), + Felt::new_unchecked(23), + Felt::new_unchecked(24), + ]); + let advice_values = vec![Felt::new_unchecked(25), Felt::new_unchecked(26)]; + let mut bytes = Vec::new(); + 0usize.write_into(&mut bytes); + 0usize.write_into(&mut bytes); + 0usize.write_into(&mut bytes); + AdviceMap::from_iter([(advice_key, advice_values)]).write_into(&mut bytes); + + let err = SparseMastForest::read_from_bytes(&bytes).unwrap_err(); + + assert!(err.to_string().contains("must not carry advice map entries")); +} + #[test] fn sparse_reader_rejects_non_increasing_full_node_ids() { let block = BasicBlockNodeBuilder::new(vec![Operation::Add]).build().unwrap(); diff --git a/core/src/mast/sparse.rs b/core/src/mast/sparse.rs index f2b9f69057..c2897bfd34 100644 --- a/core/src/mast/sparse.rs +++ b/core/src/mast/sparse.rs @@ -1,5 +1,6 @@ use alloc::{ collections::{BTreeMap, BTreeSet}, + string::ToString, sync::Arc, vec::Vec, }; @@ -85,7 +86,10 @@ impl SparseMastForest { &self.roots } - /// Returns the advice map associated with this sparse forest. + /// Returns the empty advice map associated with this sparse forest. + /// + /// Sparse replay uses `AdviceReplay` for advice reads; this map remains empty to satisfy the + /// shared [`ExecutableMastForest`] interface. pub fn advice_map(&self) -> &AdviceMap { &self.advice_map } @@ -102,6 +106,12 @@ impl SparseMastForest { roots: Vec, advice_map: AdviceMap, ) -> Result { + if !advice_map.is_empty() { + return Err(DeserializationError::InvalidValue( + "sparse MAST replay payload must not carry advice map entries".to_string(), + )); + } + let nodes = collect_unique_nodes(nodes)?; let digests = collect_unique_digests(digests)?; @@ -120,7 +130,12 @@ impl SparseMastForest { validate_full_node_child_digests(&nodes, &digests)?; - Ok(Self { nodes, digests, roots, advice_map }) + Ok(Self { + nodes, + digests, + roots, + advice_map: AdviceMap::default(), + }) } } @@ -324,8 +339,8 @@ impl SparseMastForestBuilder { } /// Consumes the builder and produces a [`SparseMastForest`] containing only the visited nodes - /// from the source forest. The roots, advice map, and debug info are cloned from the source - /// in full (they are not yet trimmed to visited nodes only). + /// from the source forest. The roots are cloned from the source in full. Advice data is not + /// copied because sparse replay uses `AdviceReplay`. pub fn finalize(self) -> SparseMastForest { let SparseMastForestBuilder { source, full_visits, digest_only_visits } = self; @@ -352,7 +367,7 @@ impl SparseMastForestBuilder { nodes, digests, roots: source.procedure_roots().to_vec(), - advice_map: source.advice_map().clone(), + advice_map: AdviceMap::default(), } } } From d094857a6c7d1fecaad8031f4c260bc8511d825c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Garillot?= Date: Fri, 10 Jul 2026 22:20:04 -0400 Subject: [PATCH 9/9] fix: reject sparse advice counts early --- core/src/mast/serialization/sparse.rs | 13 ++++++++++++- core/src/mast/serialization/tests.rs | 11 ++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/core/src/mast/serialization/sparse.rs b/core/src/mast/serialization/sparse.rs index 9408466517..cfa0f9c586 100644 --- a/core/src/mast/serialization/sparse.rs +++ b/core/src/mast/serialization/sparse.rs @@ -163,11 +163,22 @@ fn read_sparse_from( let roots = read_node_ids(source, "procedure root")?; let nodes = read_sparse_nodes(source)?; let digest_entries = read_digest_entries(source)?; - let advice_map = AdviceMap::read_from(source)?; + let advice_map = read_empty_advice_map(source)?; SparseMastForest::from_serialized_parts(nodes, digest_entries, roots, advice_map) } +fn read_empty_advice_map(source: &mut R) -> Result { + let count = source.read_usize()?; + if count != 0 { + return Err(DeserializationError::InvalidValue( + "sparse MAST replay payload must not carry advice map entries".to_string(), + )); + } + + Ok(AdviceMap::default()) +} + fn read_node_ids( source: &mut R, label: &str, diff --git a/core/src/mast/serialization/tests.rs b/core/src/mast/serialization/tests.rs index 9ddb6ac088..e584bc0d19 100644 --- a/core/src/mast/serialization/tests.rs +++ b/core/src/mast/serialization/tests.rs @@ -873,19 +873,12 @@ fn sparse_mast_drops_source_advice_map() { } #[test] -fn sparse_reader_rejects_non_empty_advice_map() { - let advice_key = Word::new([ - Felt::new_unchecked(21), - Felt::new_unchecked(22), - Felt::new_unchecked(23), - Felt::new_unchecked(24), - ]); - let advice_values = vec![Felt::new_unchecked(25), Felt::new_unchecked(26)]; +fn sparse_reader_rejects_non_empty_advice_map_count() { let mut bytes = Vec::new(); 0usize.write_into(&mut bytes); 0usize.write_into(&mut bytes); 0usize.write_into(&mut bytes); - AdviceMap::from_iter([(advice_key, advice_values)]).write_into(&mut bytes); + 1usize.write_into(&mut bytes); let err = SparseMastForest::read_from_bytes(&bytes).unwrap_err();